package com.bayer.edam.migration;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.nio.charset.Charset;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.TimeUnit;

import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.PropertySource;
import org.springframework.core.io.FileSystemResource;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.ResponseEntity;
import org.springframework.http.client.BufferingClientHttpRequestFactory;
import org.springframework.http.client.ClientHttpRequestInterceptor;
import org.springframework.http.client.SimpleClientHttpRequestFactory;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.web.client.HttpClientErrorException;
import org.springframework.web.client.HttpStatusCodeException;
import org.springframework.web.client.RestClientException;
import org.springframework.web.client.RestTemplate;

import com.bayer.edam.migration.adam.Campaign;
import com.bayer.edam.migration.adam.Classification;
import com.bayer.edam.migration.adam.ClassificationList;
import com.bayer.edam.migration.adam.EntityId;
import com.bayer.edam.migration.adam.Token;
import com.bayer.edam.migration.adam.UploadClassification;
import com.bayer.edam.migration.adam.UploadFileRecord;
import com.bayer.edam.migration.adam.UploadRecord;
import com.bayer.edam.migration.adam.UploadRecord.WriteClassification;
import com.bayer.edam.migration.adam.UploadResult;
import com.bayer.edam.migration.interceptors.LoggingRequestInterceptor;
import com.bayer.edam.migration.mam.CategoryTree;
import com.bayer.edam.migration.mam.CategoryTree.Node;
import com.bayer.edam.migration.mam.Entry;
import com.bayer.edam.migration.mam.Entry.Download;
import com.fasterxml.jackson.databind.ObjectMapper;

import net.lingala.zip4j.core.ZipFile;
import net.lingala.zip4j.model.FileHeader;

@SpringBootApplication
@PropertySource("file:secret.properties")
public class MamMigrationSelective implements CommandLineRunner {

	private static final Logger log = LoggerFactory.getLogger(MamMigrationSelective.class);
	
	static final String mamHost = "https://service.media-assistant.animalhealth.bayer.com";
	
	// XXX
	static final String fileCache = "E:\\";
	
	static final String categoryJson = "resources/2017-10-19_categories.json";
	static final String categoryMapping = "resources/Media_Assistant_Categories_131017_EDITED_without_archive.csv";
	static final String entryList = "resources/entrylist_FULL_2017-11-15.txt";

	@Value("${mam.auth}")
	private String mamAuth;

	@Value("${adam.auth}")
	private String adamAuth;

	RestTemplate restTemplate, fileTransferTemplate;
	HttpHeaders adamHeaders, adamUploadHeaders, mamHeaders;
	HttpEntity<String> adamHeadersEntity;
	
	private long overallUploadSize = 0;
	private int entryCounter = 0;
	private int skippedCounter = 0;
	private long lastLogin = 0;

	private static DecimalFormat decimalFormat = new DecimalFormat("#,##0.#");
	private SimpleDateFormat isoDate = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'");
	private SimpleDateFormat stringDate = new SimpleDateFormat("yyyy-MM-dd");
	private ObjectMapper jsonMapper = new ObjectMapper();

	private HashMap<Long, Entry> entryMap;
	private List<Classification> campaignYears;
	private List<Campaign> campaigns = new LinkedList<Campaign>();
	private CategoryTree mamCategoryTree;
	
	static final String fieldSeparator = "|~|";
	static final String lineSeparator = "\r\n";
	
	private List<UploadReference> openUploads = new ArrayList<UploadReference>();
	
	@Override
	public void run(String... strings) throws Exception {
		
		entryMap = readEntriesFromFile(entryList);
		
		List<Long> entrySelection;
		
		
		
		// XXX ----------------------------------------------------------------
		// Option 1: Process all available entries
		entrySelection = new ArrayList<Long>(entryMap.keySet());
		
		// Option 2: Process a fixed set of entry IDs
//		entrySelection = Arrays.asList(34703L, 34702L, 34701L, 34700L, 34699L, 34698L, 33626L, 33623L, 33621L, 33620L, 32781L, 31114L);
		
		// Option 3: Read entry IDs from file
//		entrySelection = new ArrayList<Long>();
//		List<String> entryIDsList = FileUtils.readLines(new File("resources/lottery.txt"), Charset.forName("utf-8"));
//		for (String idString : entryIDsList) {
//			entrySelection.add(Long.parseLong(idString));
//		}
		// --------------------------------------------------------------------
		
		Collections.sort(entrySelection);
		Collections.reverse(entrySelection);
		
		log.info("Read categories");
		mamCategoryTree = new CategoryTree(categoryJson, categoryMapping);
		
		SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory();
		requestFactory.setBufferRequestBody(false);
		fileTransferTemplate = new RestTemplate(requestFactory);
		
		restTemplate = new RestTemplate(new BufferingClientHttpRequestFactory(new SimpleClientHttpRequestFactory()));
		List<ClientHttpRequestInterceptor> interceptors = new ArrayList<ClientHttpRequestInterceptor>();
		interceptors.add(new LoggingRequestInterceptor());
		restTemplate.setInterceptors(interceptors);
		
		recreateHeaders();
		
		log.info("Read campaigns");
		campaignYears = restTemplate.exchange(Definitions.adamHost + "/classification/" + Definitions.campaignRoot + "/children", HttpMethod.GET, adamHeadersEntity, ClassificationList.class).getBody().items;
		for (Classification yearClassification : campaignYears) {
			List<Classification> campaignClassifications = restTemplate.exchange(Definitions.adamHost + "/classification/" + yearClassification.id + "/children", HttpMethod.GET, adamHeadersEntity, ClassificationList.class).getBody().items;
			for (Classification campaignClassification : campaignClassifications) {
				Campaign campaign = new Campaign(campaignClassification.name, campaignClassification.labels[0].value, yearClassification.name);
				campaign.guid = campaignClassification.id;
				campaigns.add(campaign);
			}
		}
		
//		HttpEntity<String> entity = new HttpEntity<String>("", mamHeaders);
		for (long id : entrySelection) {

			
			
			// XXX
			// if needed start from a certain ID
			if (id > 58002 || id < 35620) {
				skippedCounter++;
				continue;
			}
			
			// refresh login every 4 hours
			if (System.currentTimeMillis() - lastLogin > 14400000)
				recreateHeaders();
			
			entryCounter++;
			log.info("GET entry " + id + " (" + entryCounter + "/" + entrySelection.size() + " - " + skippedCounter + ")");

//			Entry entry = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, Entry.class, id).getBody();
			Entry entry = entryMap.get(id);
			
			log.debug("Name: " + entry.name.get("en"));
			
			// check exclusion criteria
			if ((entry.stateId == 5 || entry.stateId == 4) && entry.fileName == null && entry.categories.size() == 0) {
				log.info("RESULT Skipping entry " + entry.id + ": Status " + entry.stateId + ", no file attached, no category set");
				skippedCounter++;
				continue;
			}
			else if (entry.stateId == 7) {
				log.info("RESULT Skipping entry " + entry.id + ": Status " + entry.stateId + " (LOCKED_BY_ADMIN)");
				skippedCounter++;
				continue;
			}
			else if (entry.stateId == 8) {
				log.info("RESULT Skipping entry " + entry.id + ": Status " + entry.stateId + " (ASSIGNED_TO_INDEXATION)");
				skippedCounter++;
				continue;
			}
			else if (entry.stateId == 10) {
				log.info("RESULT Skipping entry " + entry.id + ": Status " + entry.stateId + " (ARCHIVED)");
				skippedCounter++;
				continue;
			}
			
			postToAdam(entry);
		}
		
		while (openUploads.size() > 0) {
			log.debug("Sleeping...");
			TimeUnit.SECONDS.sleep(120);
			
			log.debug("Checking back on open uploads");
			attachOpenUploads();
		}
		
		invalidateAdamToken();
	}

	void postToAdam(Entry entry) throws Exception {
		UploadRecord record = new UploadRecord();
		String entryJson = jsonMapper.writeValueAsString(entry);
		List<String> keywords = new ArrayList<String>();
		HashSet<String> keywordSet = new HashSet<String>();
		HashSet<String> campaignGuids = new HashSet<String>();
		
		// XXX
		record.addClassification("Migration.Test");
		
		// globally applicable classifications
		record.addClassification("Migration");
		record.addClassification("Country.Global");
		record.addClassification("Agencies.Bayer AH");
		record.addClassification("Usage.Restricted");
		
		// media type
		record.addClassification(Definitions.mediaTypes.get(entry.mediaType.id));
		
		// process categories
		if (entry.categories.size() == 0) {
			log.info("RESULT Skipping entry " + entry.id + ": No category assigned, cannot determine business unit and brand");
			skippedCounter++;
			return;
		}
		
		if (entry.categories.size() > 1)
			log.warn("Entry " + entry.id + " has " + entry.categories.size() + " categories assigned");
		
		for (int categoryId : entry.categories) {
			log.debug("Category: " + categoryId);
			
			Node node = mamCategoryTree.getNode(categoryId);
			log.debug("Node: " + node); 
			List<Node> path = new LinkedList<Node>();
			
			Node tmpNode = node;
			if (tmpNode.id == CategoryTree.ROOT_NODE_ID) {
				log.info("RESULT Skipping entry " + entry.id + ": Node in invisible root of category tree (" + node.toString() + ")");
				skippedCounter++;
				return;
			}
			
			String currentBrand = null;
			while (tmpNode.id != CategoryTree.ROOT_NODE_ID) {
				if (tmpNode.id == 0) {
					log.info("RESULT Skipping entry " + entry.id + ": Node outside of category tree (" + node.toString() + ")");
					skippedCounter++;
					return;
				}
				
				path.add(0, tmpNode);
				tmpNode = tmpNode.parent;
			}
			
			for (Node n : path) {
				if (n.action.equals("not migrated")) {
					log.info("RESULT Skipping entry " + entry.id + " because of tag on (parent) category " + n.name + " (" + n.id + ")");
					skippedCounter++;
					return;
				}
				
				if (n.action.equals("classify")) {
					String[] parameters = n.parameter.split(",");
					
					for (String p : parameters) {
						
						// Status "Released" is weaker than the other statuses
						if (p.startsWith("Status.")) {
							if (p.equals("Status.Released") && record.hasClassificationStartingWith("Status.")) {
								if (!record.hasClassificationStartingWith("Status.Released"))
									log.warn("Classification " + p + " not added, different status already set");
								continue;
							}
							else {
								record.removeClassificationsStartingWith("Status.");
							}
						}
						
						// Business unit "General" is weaker than the other business units
						if (p.startsWith("BusinessUnit.")) {
							if (p.equals("BusinessUnit.General") && record.hasClassificationStartingWith("BusinessUnit.")) {
								if (!record.hasClassificationStartingWith("BusinessUnit.General"))
									log.warn("Classification " + p + " not added, different business unit already set");
								continue;
							}
							else {
								record.removeClassificationsStartingWith("BusinessUnit.");
							}
						}
						
						// Brands "General" and "Others" are weaker than the other brands
						if (p.startsWith("Brand.")) {
//							if ((p.equals("Brand.General") || p.equals("Brand.CAP.Others") || p.equals("Brand.FAP.Others"))
//									&& record.hasClassificationStartingWith("Brand.")) {
//								
//								// FIXME Check if the brand matches the business unit
//								if (record.hasClassificationStartingWith("Brand.General")
//										&& !record.hasClassificationStartingWith("BusinessUnit.General")) {
//									log.warn("Classification " + p + " not added, different brand already set");
//									continue;
//								}
//							}
							
							record.removeClassificationsStartingWith("Brand.");
							currentBrand = p.substring(p.lastIndexOf('.'));
						}
						
						if (p.startsWith("MediaType.")) {
							record.removeClassificationsStartingWith("MediaType.");
						}
						
						record.addClassification(p);
					}
				}
				else if (n.action.equals("brand")) {
					record.removeClassificationsStartingWith("Brand.");
					String brandClassificationId = "Brand." + n.name.trim();
					record.addClassification(brandClassificationId);
					currentBrand = n.name.trim();
				}
				else if (n.action.equals("campaign")) {
					String campaignName = n.name;
					String campaignLabel = n.name;
					String campaignYear;
					if (n.parameter != null && !n.parameter.trim().equals("")) {
						campaignYear = n.parameter.substring(1, 5);
						if (currentBrand != null)
							campaignLabel = currentBrand + " - " + campaignLabel + " " + n.parameter;
						else
							campaignLabel = campaignLabel + " " + n.parameter;
					} else {
						campaignYear = n.name.substring(0, 4);
						if (currentBrand != null)
							campaignLabel = currentBrand + " - " + campaignLabel;
					}

					try {
						Integer.parseInt(campaignYear);
						
						// check whether the year classification already exists
						Classification yearClassification = null;
						for (Classification c : campaignYears) {
							if (c.name.equals(campaignYear)) {
								yearClassification = c;
							}
						}
						
						if (yearClassification == null) {
							log.debug("Create campaign year " + campaignYear);
							
							UploadClassification newYear = new UploadClassification(campaignYear, campaignYear);
							
							HttpEntity<UploadClassification> httpEntity = new HttpEntity<UploadClassification>(newYear, adamHeaders);
							try {
								EntityId classificationId = restTemplate.exchange(Definitions.adamHost + "/classifications", HttpMethod.POST, httpEntity, EntityId.class).getBody();
								newYear.guid = classificationId.id;
								
								// retrieve the newly created classification back
								yearClassification = restTemplate.exchange(Definitions.adamHost + "/classification/" + newYear.guid, HttpMethod.GET, adamHeadersEntity, Classification.class).getBody();
								campaignYears.add(yearClassification);
								
								log.debug("New campaign year " + campaignYear + " created: " + yearClassification.id);
							} catch (HttpStatusCodeException hsce) {
								log.error("Could not create campaign year '" + campaignYear + "' for entry " + entry.id + ": " + hsce.getStatusCode().toString());
								log.error(hsce.getResponseBodyAsString());
								hsce.printStackTrace();
							}
							
						}
						
						// check whether campaign already exists
						String campaignGuid = null;
						for (Campaign c : campaigns) {
							if (c.name.equals(campaignName)) {
								campaignGuid = c.guid;
								if (campaignGuids.add(campaignGuid)) {
									log.debug("Add existing campaign " + campaignName);
									record.addClassification(new WriteClassification(campaignGuid, 1));
								}
							}
						}
						
						if (campaignGuid == null) {
							log.info("Create campaign " + campaignYear + "/" + campaignName);
							
							Campaign campaign = new Campaign(campaignName, campaignLabel, campaignYear);
							
							HttpEntity<Campaign> httpEntity = new HttpEntity<Campaign>(campaign, adamHeaders);
							try {
								EntityId campaignId = restTemplate.exchange(Definitions.adamHost + "/classifications", HttpMethod.POST, httpEntity, EntityId.class).getBody();
								campaign.guid = campaignId.id;
								campaigns.add(campaign);
								if (campaignGuids.add(campaign.guid)) {
									log.debug("Add new campaign " + campaign.name);
									record.addClassification(new WriteClassification(campaign.guid, 1));
								}
							} 
							catch (HttpStatusCodeException hsce) {
								log.error("Could not create campaign '" + campaignName + "' for entry " + entry.id + ": " + hsce.getStatusCode().toString());
								log.error(hsce.getResponseBodyAsString());
								hsce.printStackTrace();
							}
						}
					} catch (NumberFormatException nfe) {
						log.error("Could not parse year from string " + campaignYear);
					}
				}
				else if (n.action.equals("keyword")) {
					if (keywordSet.add(n.name.trim().toLowerCase())) {
						log.debug("Adding keyword " + n.name.trim());
						keywords.add(n.name.trim());
					}
				}
			}
			
		}
		
		// Set Status.Draft for entries with stateId == 5 (Created) or 4 (Assigned)
		if (entry.stateId == 4 || entry.stateId == 5) {
			log.debug("Entry has status " + entry.stateId + ", setting record status to 'Draft'");
			record.removeClassification("Status.Released");
			record.addClassification("Status.Draft");
		}
		
		// Fields
		if (entry.name.get("en") != null) {
			record.addField("byr_Title", entry.name.get("en"));
		}
		else {
			log.warn("Entry " + entry.id + ": No title");
			record.addField("byr_Title", "No title");
		}
		
		if (entry.creationDate != null)
			record.addField("byr_ah_releaseDate", isoDate.format(entry.creationDate));
		
		record.addField("byr_ah_language", Collections.singletonList(Definitions.defaultRecordLanguage));
		
		if (entry.metaData.get("photographer") != null)
			record.addField("byr_ah_photographer", entry.metaData.get("photographer"));
		
		// Project lead
		if (entry.user != null && entry.user.get("id") != null) {
			if (Definitions.users.get(entry.user.get("id")) != null) {
				record.addField("byr_ah_projectLead", Collections.singletonList(Definitions.users.get(entry.user.get("id"))));
				record.addField("byr_ah_brandCommunicationRepresentative", Collections.singletonList(Definitions.users.get(entry.user.get("id"))));
			}
			else {
				log.warn("Entry " + entry.id + ": MAM user " + entry.user.get("id") + " not available, assigning default user");
				record.addField("byr_ah_projectLead", Collections.singletonList(Definitions.defaultProjectLead));
				record.addField("byr_ah_brandCommunicationRepresentative", Collections.singletonList(Definitions.defaultProjectLead));
			}
		}
		else {
			log.warn("Entry " + entry.id + ": No user, assigning default user");
			record.addField("byr_ah_projectLead", Collections.singletonList(Definitions.defaultProjectLead));
			record.addField("byr_ah_brandCommunicationRepresentative", Collections.singletonList(Definitions.defaultProjectLead));
		}

		// Description / Additional Information
		String description = entry.description.get("en");
		if (description == null)
			description = "";
		else
			description += "\n\n";

		if (entry.comment.get("en") != null)
			description += "Comment: " + entry.comment.get("en") + "\n\n";

		String migrationDescription = "Migrated from Media Assistant ID " + entry.id;
		if (entry.creationDate != null)
			migrationDescription += "\nDate of Issue: "	+ stringDate.format(entry.creationDate);
		
		if (entry.categories.size() == 1)
			migrationDescription += "\nCategory:";
		else
			migrationDescription += "\nCategories:";
		for (int categoryId : entry.categories) {
			migrationDescription += "\n" + mamCategoryTree.getNode(categoryId).toString();
		}
		
		
		description += migrationDescription;
		record.addField("byr_ah_description", description);

		// Keywords
		if (entry.keyword.get("en") != null && entry.keyword.get("en").length() > 0) {
			List<String> mamKeywords = Arrays.asList(entry.keyword.get("en").split("\\s*,\\s*"));
			
			for (String s : mamKeywords) {
				if (keywordSet.add(s.trim().toLowerCase())) {
					keywords.add(s.trim());
				}
					
				// if the keyword is four digits - dash - four digits, it's a GPC number
				if (s.matches("\\b\\d{4}-\\d{4}\\b"))
					record.addField("byr_ah_approvalNumber", s);
			}
		}
		
		if (keywords.size() > 0) {
			record.addField("byr_ah_keywords_tl", keywords);
		}
		
		// License Information
		record.addField("byr_ah_licenseHolderList", Collections.singletonList(Definitions.defaultLicenseHolder));
		record.addField("byr_ah_licenseAdditionalPermittedUse", Collections.singletonList(Definitions.defaultAdditionalPermittedUse));
		
		if (entry.copyright != null && entry.copyright.organization != null) {
			String licenseDescription = "";
			if (entry.copyright.use != null)
				licenseDescription +=  entry.copyright.use;
			record.addField("byr_ah_licenseComments", licenseDescription);
			
			// if license holder is not "Bayer Animal Health GmbH" put on review list
			if (!entry.copyright.organization.get("id").equals("3740"))
				log.warn("Entry " + entry.id + ": License Holder is not set to 'Bayer Animal Health GmbH' (organization 3740)");
		}
		else {
			// if license information is missing put on review list
			log.warn("Entry " + entry.id + ": Copyright information missing");
		}
		
		if (entry.gpc != null)
			record.addField("byr_ah_approvalNumber",entry.gpc.gpcNumber);
		
		// Create record
		HttpEntity<UploadRecord> httpEntity = new HttpEntity<UploadRecord>(record, adamHeaders);
		try {
			URI location = restTemplate.postForLocation(Definitions.adamHost + "/records", httpEntity, UploadRecord.class);
			log.info("RESULT Entry " + entry.id + " migrated to record " + location);

			uploadFilesForEntry(entry, location);
			log.info("FINAL " + entry.id + fieldSeparator + location.toString() + fieldSeparator + entryJson);
		} catch (HttpClientErrorException hcee) {
			log.error("RESULT Could not create record for entry " + entry.id + ", Exception: " + hcee.getMessage() + ", "
					+ hcee.getResponseBodyAsString());
		} catch (RestClientException rce) {
			log.error("RESULT Could not create record for entry " + entry.id + ", Exception: " + rce.getMessage());
			rce.printStackTrace();
		}
	}
	
	
	private void uploadFilesForEntry(Entry entry, URI recordLocation) throws Exception {
		HashSet<String> hrefsForUpload = new HashSet<String>();
		ArrayList<File> filesForUpload = new ArrayList<File>();
		
		// Select download: Download original only. If original is not available, download largest file
		Download selectedDownload = null;
		for (Download download : entry.downloadList) {
			if (download.entryFileVariation.equals("ORIGINAL")) {
				selectedDownload = download;
				break;
			}
			
			if (selectedDownload == null) {
				selectedDownload = download;
			}
			else if (selectedDownload.size < download.size) {
				selectedDownload = download;
			}
		}
		
		if (selectedDownload != null) {
			
			File masterFile = getFileForEntry(selectedDownload, entry.id);
			
			if (masterFile != null) {
				// if selected download is a zip that contains a single file, upload extracted file as master
				if (selectedDownload.fileName.toLowerCase().endsWith(".zip")) {
					ZipFile zipFile = new ZipFile(masterFile);
					
					if (!zipFile.isValidZipFile()) {
						log.error("Zip file " + masterFile.getAbsolutePath() + " could not be extracted");
					}
					else {
						
						@SuppressWarnings("unchecked")
						List<FileHeader> fileHeaders = zipFile.getFileHeaders();
						
						if (fileHeaders.size() == 1) {
							String extractedMasterFilePath = fileCache + entry.id + "\\" + fileHeaders.get(0).getFileName();
							
							if (!(new File(extractedMasterFilePath).exists())) {
								log.info("Extracting single master file " + fileCache + entry.id + "\\" + fileHeaders.get(0).getFileName());
								zipFile.extractAll(fileCache + entry.id + "\\");
							}
							log.info("Using extracted master file " + extractedMasterFilePath);
							masterFile = new File(extractedMasterFilePath);
						}
						else if (entry.mediaType.name.get("en").equals("Logo Folder")) {
							String destPath = fileCache + entry.id + "\\";
							log.info("Extracting single 'Logo Folder' files " + destPath);
							zipFile.extractAll(destPath);
							
							boolean pngFound = false;
							boolean jpgFound = false;
							String newMasterFilePath = null;
							FileHeader newMasterFileHeader = null;
							
							for (FileHeader header : fileHeaders) {
								if (header.getFileName().toLowerCase().endsWith(".png")) {
									pngFound = true;
									newMasterFilePath = destPath + header.getFileName();
									newMasterFileHeader = header;
								}
								else if (header.getFileName().toLowerCase().endsWith(".jpg") && !pngFound) {
									jpgFound = true;
									newMasterFilePath = destPath + header.getFileName();
									newMasterFileHeader = header;
								}
								else if (!header.isDirectory() && !pngFound && !jpgFound) {
									newMasterFilePath = destPath + header.getFileName();
									newMasterFileHeader = header;
								}
							}
							
							if (newMasterFilePath != null) {
								masterFile = new File(newMasterFilePath);
								log.info("Using extracted master file " + newMasterFilePath);
								
								// add remaining extracted files
								for (FileHeader header : fileHeaders) {
									if (!header.isDirectory() && header != newMasterFileHeader) {
										File extractedFile = new File(destPath + header.getFileName());
										if (extractedFile.exists())
											filesForUpload.add(extractedFile);
										else
											log.error("Could not find extraced file " + destPath + header.getFileName());
									}
								}
							}
							else {
								log.info("No suitable master file found, uploading compressed file " + masterFile.getAbsolutePath());
							}
							
						}
					}
				}
				
				log.debug("Selected master file " + masterFile.getAbsolutePath() + " for record " + recordLocation);
				filesForUpload.add(masterFile);
				hrefsForUpload.add(selectedDownload.href);
			}
			else {
				log.error("Master file " + selectedDownload.fileName + " not available");
			}
		}
		else {
			log.warn("Could not identify master file for entry " + entry.id);
		}
		
		for (Download download : entry.downloadListPublicFolder) {
			if (hrefsForUpload.add(download.href)) {
				File file = getFileForEntry(download, entry.id);
				if (file != null)
					filesForUpload.add(file);
				else
					log.error("File " + download.fileName + " not available");
			}
		}
		
		boolean isMaster = true;
		for (File file : filesForUpload) {
			if (file.exists()) {
				if (file.length() < 4000000000L) {
					log.info("Uploading file " + file.getAbsolutePath() + ", size " + file.length());
					
					MultiValueMap<String, Object> parameters = new LinkedMultiValueMap<String, Object>();
					parameters.add("file", new FileSystemResource(file));
					
					try {
						UploadResult result = fileTransferTemplate.exchange(Definitions.adamHost + "/uploads", HttpMethod.POST,
						new HttpEntity<MultiValueMap<String, Object>>(parameters, adamUploadHeaders), UploadResult.class).getBody();
						
						overallUploadSize += file.length();
						log.debug("Uploaded " + readableFileSize(overallUploadSize) + " so far");
						
						if (result.token != null) {
							log.debug("Storing upload token " + result.token + (isMaster ? " (master)" : "") + " for " + recordLocation + " from entry " + entry.id);
							UploadReference reference = new UploadReference();
							reference.targetRecord = recordLocation;
							reference.isMaster = isMaster;
							reference.token = result.token;
							reference.cachePath = file.getAbsolutePath();
							openUploads.add(reference);
						}
						else if (result.uri != null) {
							log.debug("Storing upload reference " + result.uri + (isMaster ? " (master)" : "") + " for " + recordLocation + " from entry " + entry.id);
							UploadReference reference = new UploadReference();
							reference.targetRecord = recordLocation;
							reference.isMaster = isMaster;
							reference.uri = result.uri;
							reference.cachePath = file.getAbsolutePath();
							openUploads.add(reference);
						}
						else {
							log.error("No token/URI after upload!");
						}

					} catch (HttpClientErrorException hcee) {
						log.error("FILE RESULT Could not upload " + file.getAbsolutePath() + (isMaster ? " (master)" : "") + " for " + recordLocation
								+ " from entry " + entry.id + ", Exception: " + hcee.getMessage() + ", " + hcee.getResponseBodyAsString());
					} catch (RestClientException rce) {
						log.error("FILE RESULT Could not upload " + file.getAbsolutePath() + (isMaster ? " (master)" : "") + " for " + recordLocation
								+ " from entry " + entry.id + ": " + rce.getMessage());
						rce.printStackTrace();
					}
					
				}
				else {
					// TODO
					log.error("File too large, could not upload " + file.getAbsolutePath() + (isMaster ? " (master)" : "") + " for " + recordLocation);
				}
			}
			else {
				log.error("Could not find file on disk " + file.getAbsolutePath());
			}
			
			isMaster = false;
		}
		
		// attach uploaded files to records where possible
		attachOpenUploads();
		
	}

	private void attachOpenUploads() {
		List<UploadReference> processedReferences = new ArrayList<UploadReference>();
		
		for (UploadReference reference : openUploads) {
			if (reference.token != null) {
				UploadFileRecord fileRecord = new UploadFileRecord();
				fileRecord.addFile(reference.token, reference.isMaster);
				
				log.debug("Attach " + reference.cachePath + " to " + reference.targetRecord + " via " + reference.token);
				HttpEntity<UploadFileRecord> httpEntity = new HttpEntity<UploadFileRecord>(fileRecord, adamHeaders);
				
				try {
					restTemplate.exchange(Definitions.adamHost + reference.targetRecord, HttpMethod.PUT, httpEntity, Void.class);
					log.info("ATTACH RESULT Attached " + reference.cachePath + (reference.isMaster ? " (master)" : "") + " to " + reference.targetRecord);
				} catch (HttpClientErrorException hcee) {
					log.error("ATTACH RESULT Exception while attaching file " + reference.cachePath
							+ (reference.isMaster ? " (master)" : "") + " to " + reference.targetRecord + ", Exception: "
							+ hcee.getMessage() + ", " + hcee.getResponseBodyAsString());
				} catch (RestClientException rce) {
					log.error("ATTACH RESULT Exception while attaching file " + reference.cachePath
							+ (reference.isMaster ? " (master)" : "") + " to " + reference.targetRecord + ": " + rce.getMessage());
					rce.printStackTrace();
				}
				
				processedReferences.add(reference);
			}
			else {
				log.debug("Checking status of " + reference.cachePath + (reference.isMaster ? " (master)" : "") + " for "
						+ reference.targetRecord + " at " + reference.uri);
				try {
					UploadResult result = restTemplate
							.exchange(Definitions.adamHost + reference.uri, HttpMethod.GET, adamHeadersEntity, UploadResult.class)
							.getBody();
					log.debug("Result: " + result);

					if (result.token != null) {
						log.debug("Storing upload token " + result.token);
						reference.token = result.token;
					} else if (result.status != null) {
						if (result.status.equals("Error")) {
							log.error("System could not process file " + reference.cachePath + (reference.isMaster ? " (master)" : "") + " for " + reference.targetRecord);
							processedReferences.add(reference);
						}
					}
				} catch (HttpClientErrorException hcee) {
					log.error("Failed to check status of " + reference.cachePath + (reference.isMaster ? " (master)" : "") + " for "
							+ reference.targetRecord + ", Exception: " + hcee.getMessage() + ", " + hcee.getResponseBodyAsString());
				} catch (RestClientException rce) {
					log.error("Failed to check status of " + reference.cachePath + (reference.isMaster ? " (master)" : "") + " for "
							+ reference.targetRecord + ": " + rce.getMessage());
				}
			}
		}
		
		openUploads.removeAll(processedReferences);
		log.info(openUploads.size() + " open uploads remaining");
		
		for (UploadReference reference : openUploads) {
			log.debug("    " + reference.uri + " for " + reference.targetRecord + (reference.isMaster ? " (master)" : "") + " as " + reference.cachePath);
		}
	}

	private File getFileForEntry(Download download, Long id) throws IOException {
		String filenameToUse;
		if (download.fileNameOnDisc != null)
			filenameToUse = download.fileNameOnDisc;
		else
			filenameToUse = download.fileName;
		
		File file = new File(fileCache + id + "\\" + filenameToUse);
		
		// check whether file is in cache, download if it's not there
		if (!file.exists()) {
			try {
				log.debug("Downloading missing file " + filenameToUse + " for entry " + id + " at " + download.href);
				URL url = new URL(download.href);
				HttpURLConnection httpConn = (HttpURLConnection) url.openConnection();
				httpConn.setRequestProperty("Authorization", "Basic " + mamAuth);
				ReadableByteChannel rbc = Channels.newChannel(httpConn.getInputStream());
				FileOutputStream fos = new FileOutputStream(fileCache + id + "\\" + filenameToUse);
				fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
				fos.close();
			}
			catch (IOException ioe) {
				log.error("Could not download file " + filenameToUse + " for entry " + id + ": " + ioe.getMessage());
				return null;
			}
		}
		else {
			log.debug("Using existing file " + filenameToUse + " for entry " + id);
		}
		
		return file;
	}

	private HashMap<Long, Entry> readEntriesFromFile(String filePath) throws IOException {
		File entryListFile = new File(filePath);
		log.debug("Reading entries from " + entryListFile.getAbsolutePath());
		List<String> entryList = FileUtils.readLines(entryListFile, Charset.forName("utf-8"));
		HashMap<Long, Entry> entryMap = new HashMap<Long, Entry>();
		
		for (String entryLine : entryList) {
			
			String entryIdString = entryLine.substring(0, entryLine.indexOf(fieldSeparator));
			Long entryId = Long.parseLong(entryIdString);
			
			String entryJson = entryLine.substring(entryLine.indexOf(fieldSeparator) + 3);
			Entry entry = jsonMapper.readValue(entryJson, Entry.class);
			
			entryMap.put(entryId, entry);
		}
		
		return entryMap;
	}
	
	private void recreateHeaders() {
		adamHeaders = createAdamHeaders();
		adamHeadersEntity = new HttpEntity<String>("", adamHeaders);
		adamUploadHeaders = createAdamHeaders();
		adamUploadHeaders.set("Content-Type", "multipart/form-data");
		adamUploadHeaders.set("Accept", "*/*");
		mamHeaders = createMamHeaders();
		
		lastLogin = System.currentTimeMillis();
	}

	private HttpHeaders createMamHeaders() {
		log.debug("MediaAssistant login");
		
		HttpHeaders headers = new HttpHeaders();
		headers.add("Accept", "application/json");
		headers.add("Authorization", "Basic " + mamAuth);
		headers.add("Content-Type", "application/json");
		headers.add("From", "migration@bayer.com");

		return headers;
	}

	private HttpHeaders createAdamHeaders() {
		log.debug("Adam login");
		
		HttpHeaders headers = new HttpHeaders();
		headers.add("Registration", "ADAMBAYER");
		headers.add("API-VERSION", "1");
		headers.add("Accept", "application/hal+json");
		headers.add("Content-Type", "application/json");
		headers.add("Authorization", "Basic " + adamAuth);
		headers.add("set-immediateSearchIndexUpdate", "true"); // see https://edam.bayer.com:2015/docs/resources/record#example-records-direct-push
		HttpEntity<String> entity = new HttpEntity<String>("", headers);
		Token token = restTemplate.exchange(Definitions.adamHost + "/auth", HttpMethod.GET, entity, Token.class).getBody();

		// replace auth header
		headers.remove("Authorization");
		headers.add("Authorization", "Token " + token.getToken());

		// replace accept-charset header (in order to declutter log)
		headers.add("Accept-Charset", "us-ascii, utf-16, utf-8");

		return headers;
	}

	private void invalidateAdamToken() {
		ResponseEntity<String> response = restTemplate.exchange(Definitions.adamHost + "/auth", HttpMethod.DELETE, adamHeadersEntity, String.class);
		log.info("Logged out from adam API: " + response.toString());
	}
	
	/*
	 * // Quoted "Z" to indicate UTC, no timezone offset private static SimpleDateFormat outputDateFormat = new
	 * SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'"); private static SimpleDateFormat inputDateFormat = new SimpleDateFormat("yyyy-MM-dd");
	 * 
	 * public static String toIso8601(String isoDate) { try { return outputDateFormat.format(inputDateFormat.parse(isoDate)); } catch
	 * (ParseException e) { log.error(e.toString()); e.printStackTrace(); } return null; }
	 */
	
	public static String readableFileSize(long size) {
	    if(size <= 0) return "0";
	    final String[] units = new String[] { "B", "kB", "MB", "GB", "TB" };
	    int digitGroups = (int) (Math.log10(size)/Math.log10(1024));
	    return decimalFormat.format(size/Math.pow(1024, digitGroups)) + " " + units[digitGroups];
	}

	public static void main(String args[]) {
		SpringApplication.run(MamMigrationSelective.class, args);
	}

}