EntryDownloader - download all entry IDs
This commit is contained in:
gexce
@@ -8,10 +8,12 @@ import java.net.PasswordAuthentication;
|
||||
import java.net.URL;
|
||||
import java.nio.channels.Channels;
|
||||
import java.nio.channels.ReadableByteChannel;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
@@ -29,12 +31,20 @@ import com.bayer.edam.migration.mam.Entry.Download;
|
||||
import com.bayer.edam.migration.mam.Entrylist;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
//@SpringBootApplication
|
||||
@SpringBootApplication
|
||||
@PropertySource("file:secret.properties")
|
||||
public class EntryDownloader implements CommandLineRunner {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(EntryDownloader.class);
|
||||
|
||||
/**
|
||||
* If true, files are downloaded. If false, only the IDs are written into the outputfile.
|
||||
*/
|
||||
private final boolean downloadFiles = false;
|
||||
|
||||
File outputfile = new File("C:\\Temp\\Migration AH\\files\\entrylist_2017-10-25.txt");
|
||||
|
||||
|
||||
@Value("${mam.auth}")
|
||||
private String mamAuth;
|
||||
|
||||
@@ -82,81 +92,94 @@ public class EntryDownloader implements CommandLineRunner {
|
||||
for (int pageNum = 1; pageNum <= lastPage; pageNum++) {
|
||||
Entrylist entrylist = restTemplate.exchange(mamHost + "/rest/migration/entrylist?pagenumber={pagenum}&pagesize=500", HttpMethod.GET, entity, Entrylist.class, pageNum).getBody();
|
||||
lastPage = entrylist.lastPage;
|
||||
log.debug("processing page " + pageNum + " of " + lastPage);
|
||||
|
||||
for (int i = 0; i < entrylist.entries.length; i++) {
|
||||
|
||||
Integer entryId = Integer.parseInt(entrylist.entries[i].mediaAssistantId);
|
||||
File targetDir = new File("E:\\" + entryId);
|
||||
if (targetDir.exists()) {
|
||||
log.info("Skip " + entryId + ", target directory already present");
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
String entryJson = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, String.class, entrylist.entries[i].mediaAssistantId).getBody();
|
||||
Entry entry = jsonMapper.readValue(entryJson, Entry.class);
|
||||
|
||||
List<Download> downloadSelection = new ArrayList<Download>();
|
||||
|
||||
// Select download: Download original only. If original is not available, download largest file
|
||||
Download selectedDownload = null;
|
||||
for (Download download : entry.downloadList) {
|
||||
if (download.entryFileVariation.equals("ORIGINAL")) {
|
||||
selectedDownload = download;
|
||||
break;
|
||||
}
|
||||
|
||||
if (selectedDownload == null) {
|
||||
selectedDownload = download;
|
||||
}
|
||||
else if (selectedDownload.size < download.size) {
|
||||
selectedDownload = download;
|
||||
}
|
||||
}
|
||||
|
||||
if (selectedDownload != null)
|
||||
downloadSelection.add(selectedDownload);
|
||||
|
||||
// TODO extract, if zip --> postponed until upload
|
||||
|
||||
for (Download download : entry.downloadListPublicFolder) {
|
||||
downloadSelection.add(download);
|
||||
}
|
||||
|
||||
// clean download selection
|
||||
HashSet<String> hrefs = new HashSet<String>();
|
||||
List<Download> cleanedDownloads = new ArrayList<Download>();
|
||||
for (Download download : downloadSelection) {
|
||||
if (hrefs.add(download.href)) {
|
||||
cleanedDownloads.add(download);
|
||||
}
|
||||
}
|
||||
|
||||
long aggregatedFilesize = 0;
|
||||
for (Download download : cleanedDownloads) {
|
||||
aggregatedFilesize += download.size;
|
||||
}
|
||||
|
||||
log.info(entry.id + separator + entry.mediaAssistantId + separator + entry.stateId + separator + aggregatedFilesize + separator + entryJson);
|
||||
|
||||
targetDir.mkdir();
|
||||
|
||||
for (Download download : cleanedDownloads) {
|
||||
String filenameToUse;
|
||||
if (download.fileNameOnDisc != null)
|
||||
filenameToUse = download.fileNameOnDisc;
|
||||
else
|
||||
filenameToUse = download.fileName;
|
||||
|
||||
URL fileLink = new URL(download.href);
|
||||
ReadableByteChannel rbc = Channels.newChannel(fileLink.openStream());
|
||||
FileOutputStream fos = new FileOutputStream("E:\\" + entry.id + "\\" + filenameToUse);
|
||||
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
|
||||
fos.close();
|
||||
if (!downloadFiles) {
|
||||
// only create a list with all IDs
|
||||
try {
|
||||
FileUtils.writeStringToFile(outputfile, entrylist.entries[i].mediaAssistantId + "\n", Charset.forName("utf-8"), true);
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
catch (IOException ioe) {
|
||||
log.error(ioe.getMessage());
|
||||
else {
|
||||
|
||||
Integer entryId = Integer.parseInt(entrylist.entries[i].mediaAssistantId);
|
||||
File targetDir = new File("E:\\" + entryId);
|
||||
if (targetDir.exists()) {
|
||||
log.info("Skip " + entryId + ", target directory already present");
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
String entryJson = restTemplate.exchange(mamHost + "/rest/migration/entry/{id}", HttpMethod.GET, entity, String.class, entrylist.entries[i].mediaAssistantId).getBody();
|
||||
Entry entry = jsonMapper.readValue(entryJson, Entry.class);
|
||||
|
||||
List<Download> downloadSelection = new ArrayList<Download>();
|
||||
|
||||
// Select download: Download original only. If original is not available, download largest file
|
||||
Download selectedDownload = null;
|
||||
for (Download download : entry.downloadList) {
|
||||
if (download.entryFileVariation.equals("ORIGINAL")) {
|
||||
selectedDownload = download;
|
||||
break;
|
||||
}
|
||||
|
||||
if (selectedDownload == null) {
|
||||
selectedDownload = download;
|
||||
}
|
||||
else if (selectedDownload.size < download.size) {
|
||||
selectedDownload = download;
|
||||
}
|
||||
}
|
||||
|
||||
if (selectedDownload != null)
|
||||
downloadSelection.add(selectedDownload);
|
||||
|
||||
// TODO extract, if zip --> postponed until upload
|
||||
|
||||
for (Download download : entry.downloadListPublicFolder) {
|
||||
downloadSelection.add(download);
|
||||
}
|
||||
|
||||
// clean download selection
|
||||
HashSet<String> hrefs = new HashSet<String>();
|
||||
List<Download> cleanedDownloads = new ArrayList<Download>();
|
||||
for (Download download : downloadSelection) {
|
||||
if (hrefs.add(download.href)) {
|
||||
cleanedDownloads.add(download);
|
||||
}
|
||||
}
|
||||
|
||||
long aggregatedFilesize = 0;
|
||||
for (Download download : cleanedDownloads) {
|
||||
aggregatedFilesize += download.size;
|
||||
}
|
||||
|
||||
log.info(entry.id + separator + entry.mediaAssistantId + separator + entry.stateId + separator + aggregatedFilesize + separator + entryJson);
|
||||
|
||||
targetDir.mkdir();
|
||||
|
||||
for (Download download : cleanedDownloads) {
|
||||
String filenameToUse;
|
||||
if (download.fileNameOnDisc != null)
|
||||
filenameToUse = download.fileNameOnDisc;
|
||||
else
|
||||
filenameToUse = download.fileName;
|
||||
|
||||
URL fileLink = new URL(download.href);
|
||||
ReadableByteChannel rbc = Channels.newChannel(fileLink.openStream());
|
||||
FileOutputStream fos = new FileOutputStream("E:\\" + entry.id + "\\" + filenameToUse);
|
||||
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
|
||||
fos.close();
|
||||
}
|
||||
}
|
||||
catch (IOException ioe) {
|
||||
log.error(ioe.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user