mam-migration/AH MAM Migration/src/com/bayer/edam/migration/EntryDownloader.java

package com.bayer.edam.migration;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.Authenticator;
import java.net.PasswordAuthentication;
import java.net.URL;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.concurrent.TimeUnit;

import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.context.annotation.PropertySource;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.web.client.RestTemplate;

import com.bayer.edam.migration.mam.Entry;
import com.bayer.edam.migration.mam.Entry.Download;
import com.bayer.edam.migration.mam.Entrylist;
import com.fasterxml.jackson.databind.ObjectMapper;

//@SpringBootApplication
@PropertySource("file:secret.properties")
public class EntryDownloader implements CommandLineRunner {
	
	private static final Logger log = LoggerFactory.getLogger(EntryDownloader.class);
	
	/**
	 * If true, files are downloaded. If false, only the IDs are written into the outputfile.
	 */
	private final boolean downloadFiles = false;
	
	File outputfile = new File("C:\\Temp\\Migration AH\\files\\entrylist_2017-11-09.txt");

	
	@Value("${mam.auth}")
	private String mamAuth;
	
	@Value("${mam.user}")
	private String mamUser;
	
	@Value("${mam.password}")
	private String mamPassword;

	
	static final String mamHost = "https://service.media-assistant.animalhealth.bayer.com";
	
	static final String separator = "|~|";

	
	private ObjectMapper jsonMapper = new ObjectMapper();
	private RestTemplate restTemplate = new RestTemplate();
	
	public static class MamAuthenticator extends Authenticator {
		private final PasswordAuthentication auth;
		
		public MamAuthenticator(String mamUser, String mamPassword) {
			auth = new PasswordAuthentication(mamUser, mamPassword.toCharArray());
		}

		@Override
		protected PasswordAuthentication getPasswordAuthentication() {
			return auth;
		}
	}
	
	
	@Override
	public void run(String... strings) {
		HttpHeaders headers = new HttpHeaders();
		headers.add("Accept", "application/json");
		headers.add("Authorization", "Basic " + mamAuth);
		headers.add("Content-Type", "application/json");
		headers.add("From", "migration@bayer.com");
		HttpEntity<String> entity = new HttpEntity<String>("", headers);
		
		Authenticator.setDefault(new MamAuthenticator(mamUser, mamPassword));
		
		int lastPage = 1;
		for (int pageNum = 1; pageNum <= lastPage; pageNum++) {
			Entrylist entrylist = restTemplate.exchange(mamHost + "/rest/migration/entrylist?pagenumber={pagenum}&pagesize=500", HttpMethod.GET, entity, Entrylist.class, pageNum).getBody();
			lastPage = entrylist.lastPage;
			log.debug("processing page " + pageNum + " of " + lastPage);
			
			for (int i = 0; i < entrylist.entries.length; i++) {
				
				if (!downloadFiles) {
					// only create a list with all IDs
					try {
						log.info("Get " + i + "/" + entrylist.entries.length);
						TimeUnit.SECONDS.sleep(1); // XXX
						int id = Integer.parseInt(entrylist.entries[i].mediaAssistantId);
						Entry entry = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, Entry.class, id).getBody();
						String entryJson = jsonMapper.writeValueAsString(entry);
						FileUtils.writeStringToFile(outputfile, id + separator + entryJson + "\n", Charset.forName("utf-8"), true);
					} catch (Exception e) {
						log.error("Could not get entry " + entrylist.entries[i].mediaAssistantId);
						e.printStackTrace();
					}
				}
				else {
				
					Integer entryId = Integer.parseInt(entrylist.entries[i].mediaAssistantId);
					
					File targetDir = new File("E:\\" + entryId);
					if (targetDir.exists()) {
						log.info("Skip " + entryId + ", target directory already present");
						continue;
					}
					
					try {
						String entryJson = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, String.class, entrylist.entries[i].mediaAssistantId).getBody();
						Entry entry = jsonMapper.readValue(entryJson, Entry.class);
						
						List<Download> downloadSelection = new ArrayList<Download>();
						
						// Select download: Download original only. If original is not available, download largest file
						Download selectedDownload = null;
						for (Download download : entry.downloadList) {
							if (download.entryFileVariation.equals("ORIGINAL")) {
								selectedDownload = download;
								break;
							}
							
							if (selectedDownload == null) {
								selectedDownload = download;
							}
							else if (selectedDownload.size < download.size) {
								selectedDownload = download;
							}
						}
						
						if (selectedDownload != null)
							downloadSelection.add(selectedDownload);
						
						// TODO extract, if zip --> postponed until upload
						
						for (Download download : entry.downloadListPublicFolder) {
							downloadSelection.add(download);
						}
						
						// clean download selection
						HashSet<String> hrefs = new HashSet<String>();
						List<Download> cleanedDownloads = new ArrayList<Download>();
						for (Download download : downloadSelection) {
							if (hrefs.add(download.href)) {
								cleanedDownloads.add(download);
							}
						}
						
						long aggregatedFilesize = 0;
						for (Download download : cleanedDownloads) {
							aggregatedFilesize += download.size;
						}
						
						log.info(entry.id + separator + entry.mediaAssistantId + separator + entry.stateId + separator + aggregatedFilesize + separator + entryJson);
						
						targetDir.mkdir();
						
						for (Download download : cleanedDownloads) {
							String filenameToUse;
							if (download.fileNameOnDisc != null)
								filenameToUse = download.fileNameOnDisc;
							else
								filenameToUse = download.fileName;
							
							URL fileLink = new URL(download.href);
							ReadableByteChannel rbc = Channels.newChannel(fileLink.openStream());
							FileOutputStream fos = new FileOutputStream("E:\\" + entry.id + "\\" + filenameToUse);
							fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
							fos.close();
						}
					}
					catch (IOException ioe) {
						log.error(ioe.getMessage());
					}
				}
			}
		}
	}

	public static void main(String args[]) {
		SpringApplication.run(EntryDownloader.class, args);
	}
	
}