package com.bayer.edam.migration; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.net.Authenticator; import java.net.PasswordAuthentication; import java.net.URL; import java.nio.channels.Channels; import java.nio.channels.ReadableByteChannel; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.concurrent.TimeUnit; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.CommandLineRunner; import org.springframework.boot.SpringApplication; import org.springframework.context.annotation.PropertySource; import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; import org.springframework.web.client.RestTemplate; import com.bayer.edam.migration.mam.Entry; import com.bayer.edam.migration.mam.Entry.Download; import com.bayer.edam.migration.mam.Entrylist; import com.fasterxml.jackson.databind.ObjectMapper; //@SpringBootApplication @PropertySource("file:secret.properties") public class EntryDownloader implements CommandLineRunner { private static final Logger log = LoggerFactory.getLogger(EntryDownloader.class); /** * If true, files are downloaded. If false, only the IDs are written into the outputfile. */ private final boolean downloadFiles = false; File outputfile = new File("C:\\Temp\\Migration AH\\files\\entrylist_2017-11-09.txt"); @Value("${mam.auth}") private String mamAuth; @Value("${mam.user}") private String mamUser; @Value("${mam.password}") private String mamPassword; static final String mamHost = "https://service.media-assistant.animalhealth.bayer.com"; static final String separator = "|~|"; private ObjectMapper jsonMapper = new ObjectMapper(); private RestTemplate restTemplate = new RestTemplate(); public static class MamAuthenticator extends Authenticator { private final PasswordAuthentication auth; public MamAuthenticator(String mamUser, String mamPassword) { auth = new PasswordAuthentication(mamUser, mamPassword.toCharArray()); } @Override protected PasswordAuthentication getPasswordAuthentication() { return auth; } } @Override public void run(String... strings) { HttpHeaders headers = new HttpHeaders(); headers.add("Accept", "application/json"); headers.add("Authorization", "Basic " + mamAuth); headers.add("Content-Type", "application/json"); headers.add("From", "migration@bayer.com"); HttpEntity entity = new HttpEntity("", headers); Authenticator.setDefault(new MamAuthenticator(mamUser, mamPassword)); int lastPage = 1; for (int pageNum = 1; pageNum <= lastPage; pageNum++) { Entrylist entrylist = restTemplate.exchange(mamHost + "/rest/migration/entrylist?pagenumber={pagenum}&pagesize=500", HttpMethod.GET, entity, Entrylist.class, pageNum).getBody(); lastPage = entrylist.lastPage; log.debug("processing page " + pageNum + " of " + lastPage); for (int i = 0; i < entrylist.entries.length; i++) { if (!downloadFiles) { // only create a list with all IDs try { log.info("Get " + i + "/" + entrylist.entries.length); TimeUnit.SECONDS.sleep(1); // XXX int id = Integer.parseInt(entrylist.entries[i].mediaAssistantId); Entry entry = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, Entry.class, id).getBody(); String entryJson = jsonMapper.writeValueAsString(entry); FileUtils.writeStringToFile(outputfile, id + separator + entryJson + "\n", Charset.forName("utf-8"), true); } catch (Exception e) { log.error("Could not get entry " + entrylist.entries[i].mediaAssistantId); e.printStackTrace(); } } else { Integer entryId = Integer.parseInt(entrylist.entries[i].mediaAssistantId); File targetDir = new File("E:\\" + entryId); if (targetDir.exists()) { log.info("Skip " + entryId + ", target directory already present"); continue; } try { String entryJson = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, String.class, entrylist.entries[i].mediaAssistantId).getBody(); Entry entry = jsonMapper.readValue(entryJson, Entry.class); List downloadSelection = new ArrayList(); // Select download: Download original only. If original is not available, download largest file Download selectedDownload = null; for (Download download : entry.downloadList) { if (download.entryFileVariation.equals("ORIGINAL")) { selectedDownload = download; break; } if (selectedDownload == null) { selectedDownload = download; } else if (selectedDownload.size < download.size) { selectedDownload = download; } } if (selectedDownload != null) downloadSelection.add(selectedDownload); // TODO extract, if zip --> postponed until upload for (Download download : entry.downloadListPublicFolder) { downloadSelection.add(download); } // clean download selection HashSet hrefs = new HashSet(); List cleanedDownloads = new ArrayList(); for (Download download : downloadSelection) { if (hrefs.add(download.href)) { cleanedDownloads.add(download); } } long aggregatedFilesize = 0; for (Download download : cleanedDownloads) { aggregatedFilesize += download.size; } log.info(entry.id + separator + entry.mediaAssistantId + separator + entry.stateId + separator + aggregatedFilesize + separator + entryJson); targetDir.mkdir(); for (Download download : cleanedDownloads) { String filenameToUse; if (download.fileNameOnDisc != null) filenameToUse = download.fileNameOnDisc; else filenameToUse = download.fileName; URL fileLink = new URL(download.href); ReadableByteChannel rbc = Channels.newChannel(fileLink.openStream()); FileOutputStream fos = new FileOutputStream("E:\\" + entry.id + "\\" + filenameToUse); fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); fos.close(); } } catch (IOException ioe) { log.error(ioe.getMessage()); } } } } } public static void main(String args[]) { SpringApplication.run(EntryDownloader.class, args); } }