Skip to content

Commit

Permalink
#12 APPROVE - matching logic. Some refactoring.
Browse files Browse the repository at this point in the history 
#18 IMPORT - Thumb generation fixed
  • Loading branch information
@spuliaiev-sfdc
spuliaiev-sfdc committed Jul 11, 2018
1 parent 8d63301 commit 223d0b0
Show file tree
Hide file tree
Showing 19 changed files with 17,249 additions and 124 deletions.
2 changes: 2 additions & 0 deletions core/dao/src/main/java/gallerymine/backend/beans/repository/ImportRequestRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ public interface ImportRequestRepository extends MongoRepository<ImportRequest,

Page<ImportRequest> findByParent(String parent, Pageable pageable);

Page<ImportRequest> findByParentAndIndexProcessIds(String parent, String processId, Pageable pageable);

Collection<ImportRequest> findByParent(String parent);

Page<ImportRequest> findByParentNull(Pageable pageable);
Expand Down
4 changes: 2 additions & 2 deletions ...ibs/image-importer/src/main/java/gallerymine/backend/importer/ImportApproveProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ public ImportApproveProcessor() {
}

public void requestProcessing(ImportRequest request, Process process) throws ImportFailedException {
log.warn(this.getClass().getSimpleName()+" approve processing id={} status={} path={}", request.getId(), request.getStatus(), request.getPath());
log.warn(" approve processing start id={} status={} path={}", request.getId(), request.getStatus(), request.getPath());
ImportRequest.ImportStats stats = request.getStats(processType);
ImportRequest.ImportStats statsEnum = request.getStats(ProcessType.MATCHING);
stats.setFolders(statsEnum.getFolders());
Expand All @@ -65,7 +65,7 @@ public void requestProcessing(ImportRequest request, Process process) throws Imp
requestRepository.save(request);
}

log.info(this.getClass().getSimpleName()+" matching processing id={} status={} path={}", request.getId(), request.getStatus(), request.getPath());
log.info(" approve processing done id={} status={} path={}", request.getId(), request.getStatus(), request.getPath());
}

}
13 changes: 9 additions & 4 deletions ...bs/image-importer/src/main/java/gallerymine/backend/importer/ImportMatchingProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ public ImportMatchingProcessor() {
}

public void requestProcessing(ImportRequest request, Process process) throws ImportFailedException {
log.warn(" matching processing start id={} status={} path={}", request.getId(), request.getStatus(), request.getPath());
Path path = appConfig.getImportRootFolderPath().resolve(request.getPath());

if (!validateImportRequest(process, path))
Expand All @@ -69,7 +70,7 @@ public void requestProcessing(ImportRequest request, Process process) throws Imp
stats.setFolders(statsEnum.getFolders());
stats.setFiles(statsEnum.getFilesDone());
requestRepository.save(request);
log.info(this.getClass().getSimpleName()+" matching processing id={} status={} path={}", request.getId(), request.getStatus(), request.getPath());
log.info(" matching processing id={} status={} path={}", request.getId(), request.getStatus(), request.getPath());
try {
int filesCount = 0;
int filesSucceedCount = 0;
Expand All @@ -96,23 +97,27 @@ public void requestProcessing(ImportRequest request, Process process) throws Imp
.incFilesDone();
} catch (Exception e) {
request.getStats(processType).incFailed();
log.error(this.getClass().getSimpleName()+" Failed processing info id=%s path=%s", info.getId(), info.getFileName());
log.error(" matching processing failed: Failed processing info id={} path={}", info.getId(), info.getFileName());
}
log.info(" matching processing done {} or {} succeeded. id={} status={} path={}",
filesCount, filesSucceedCount,
request.getId(), request.getStatus(), request.getPath());
requestRepository.save(request);
}

String info = request.addNote("Matching info gathered for id=%s files %d of %d. Failed:%d",
request.getId(), filesSucceedCount, filesCount, filesCount-filesSucceedCount);
log.info(this.getClass().getSimpleName()+" "+info);
log.info(" "+info);

request.setStatus(statusHolder.getProcessingDone());
request.getStats(processType).setAllFilesProcessed(true);
requestRepository.save(request);
} catch (Exception e) {
request.addError("Matching info analysing failed for indexRequest id=%s", request.getId());
requestRepository.save(request);
log.error(this.getClass().getSimpleName()+" Matching info analysing failed for indexRequest id=%s {}. Reason: {}", path, e.getMessage());
log.error(" matching processing failed: Matching info analysing failed for indexRequest id=%s {}. Reason: {}", path, e.getMessage());
}
log.warn(" matching processing done id={} status={} path={}", request.getId(), request.getStatus(), request.getPath());
}

}
31 changes: 22 additions & 9 deletions core/libs/image-importer/src/main/java/gallerymine/backend/importer/ImportProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,16 +122,19 @@ public ImportRequest registerImport(Path originalPath, boolean enforceImport) {
} */

public void requestProcessing(ImportRequest requestSrc, Process process) throws ImportFailedException {
log.warn(" import processing start id={} status={} path={}", request.getId(), request.getStatus(), request.getPath());
Path path = appConfig.getImportRootFolderPath().resolve(request.getPath());

if (!validateImportRequest(process, path))
return;

ImportRequest.ImportStatus oldStatus = request.getStatus();
request.setStatus(statusHolder.getInProcessing());
request.getStats(processType).getFiles().set(-1);
requestRepository.save(request);

Path enumeratingDir = null;
log.info(" folders enumerating id={} status={} path={}", request.getId(), request.getStatus(), request.getPath());
try (DirectoryStream<Path> directoryStreamOfFolders = Files.newDirectoryStream(path, file -> file.toFile().isDirectory())) {
int foldersCount = 0;
for (Path dir : directoryStreamOfFolders) {
Expand All @@ -143,26 +146,29 @@ public void requestProcessing(ImportRequest requestSrc, Process process) throws
request.setFoldersCount(foldersCount);
request.getStats(processType).incFolders(foldersCount);
requestRepository.save(request);
log.info("ImportRequest status changed id={} status={} path={}", request.getId(), request.getStatus(), request.getPath());
log.info(" folders processed {}. id={} status={} path={}",
foldersCount,
request.getId(), request.getStatus(), request.getPath());
} catch (IOException e) {
request.setFoldersCount(-1);
request.addError("ImportRequest indexing failed for folder "+enumeratingDir);
requestRepository.save(request);
log.error("ImportRequest Failed to index. id={} status={} path={} reason='{}'", request.getId(), request.getStatus(), request.getPath(), e.getMessage(), e);
log.error(" Failed to index. id={} status={} path={} reason='{}'", request.getId(), request.getStatus(), request.getPath(), e.getMessage(), e);
}

log.info(" files enumerating id={} status={} path={}", request.getId(), request.getStatus(), request.getPath());
try {
log.info("ImportRequest status changed id={} status={} path={}", request.getId(), request.getStatus(), request.getPath());
Path importRootFolder = Paths.get(request.getRootPath());
int filesCount = 0;
int filesIgnoredCount = 0;
int filesSucceedCount = 0;
ImportRequest.ImportStats stats = request.getStats(processType);
try (DirectoryStream<Path> directoryStreamOfFiles = Files.newDirectoryStream(path, file -> file.toFile().isFile())) {
for (Path file : directoryStreamOfFiles) {
String fileName = file.toString().toLowerCase();
String fileExt = fileName.substring(fileName.lastIndexOf(".") + 1);
if (fileName.startsWith(".")) {
log.info("ImportRequest Ignore system file {}", file.toAbsolutePath().toString());
log.info(" Ignore system file {}", file.toAbsolutePath().toString());
continue;
}

Expand All @@ -183,14 +189,15 @@ public void requestProcessing(ImportRequest requestSrc, Process process) throws
stats.incMovedToApprove();
info.setRootPath(targetFolder.toFile().getAbsolutePath());
info.setStatus(InfoStatus.ANALYSING);
filesSucceedCount++;
}
} else {
filesIgnoredCount++;
stats.incFailed();
Path targetFolder = importUtils.moveToFailed(file, request.getRootPath());
info.setRootPath(targetFolder.toFile().getAbsolutePath());
info.setStatus(InfoStatus.FAILED);
logUnknownFormats.error("ImportRequest Unknown format of file {}", file.toAbsolutePath().toString());
logUnknownFormats.error(" Unknown format of file {}", file.toAbsolutePath().toString());
}
} else {
Path targetFolder = importUtils.moveToFailed(file, request.getRootPath());
Expand All @@ -200,7 +207,7 @@ public void requestProcessing(ImportRequest requestSrc, Process process) throws
}
importSourceRepository.saveByGrade(info);
if (!InfoStatus.FAILED.equals(info.getStatus()) && !info.hasThumb()) {
log.warn(" No thumbnail injected - need to generate one for {} in {}", info.getFileName(), info.getFilePath());
log.warn(" No thumbnail injected - need to generate one for {} in {}", info.getFileName(), info.getFilePath());
Path thumbStoredFile = importUtils.generatePicThumbName(info.getFileName(), info.getTimestamp());
String relativeStoredPath = appConfig.relativizePathToThumb(thumbStoredFile.toFile().getAbsolutePath());
ThumbRequest request = new ThumbRequest(info.getFullFilePath(), relativeStoredPath);
Expand All @@ -215,19 +222,23 @@ public void requestProcessing(ImportRequest requestSrc, Process process) throws
stats.setAllFilesProcessed(true);
stats.getFiles().set(filesCount);
if (stats.getFilesDone().get() != filesCount) {
log.warn("Cound Mismatch for ImportRequest id={} done={} <> {}",
log.warn(" Count Mismatch for ImportRequest id={} done={} <> {}",
request.getId(), stats.getFilesDone().get() , filesCount);
}
stats.getFilesDone().set(filesCount);
requestRepository.save(request);
log.info(" files processing done {} or {} succeeded ({} ignored). id={} status={} path={}",
filesCount, filesSucceedCount, filesIgnoredCount,
request.getId(), request.getStatus(), request.getPath());
} catch (IOException e) {
request.setFilesCount(-1);
request.setFilesIgnoredCount(-1);
request.getStats(processType).setAllFilesProcessed(true);
request.addError("indexing failed for file "+path);
requestRepository.save(request);
log.error("ImportRequest indexing failed for file {}. Reason: {}", path, e.getMessage());
log.error(" import processing failed for file {}. Reason: {}", path, e.getMessage());
}
log.warn(" import processing done id={} status={} path={}", request.getId(), request.getStatus(), request.getPath());
}

/**
Expand Down Expand Up @@ -285,6 +296,7 @@ public ImportRequest registerNewImportFolderRequest(String path, ImportRequest p
} catch (FileNotFoundException e) {
throw new ImportFailedException("File not found "+ path, e);
}
log.info(" register new Import path={} processId={} {}", path, indexProcessId, parent != null ? ("path="+parent.getPath()) : "");
ImportRequest newRequest = requestRepository.findByPath(path);
if (newRequest == null) {
newRequest = new ImportRequest();
Expand All @@ -308,7 +320,8 @@ public ImportRequest registerNewImportFolderRequest(String path, ImportRequest p
newRequest.setRootId(newRequest.getId());
}
requestRepository.save(newRequest);
log.info("ImportRequest status changed id={} status={} path={}", newRequest.getId(), newRequest.getStatus(), newRequest.getPath());
log.info(" registered as id={} status={} path={} processid={} {}", newRequest.getId(), newRequest.getStatus(), newRequest.getPath(),
indexProcessId, parent != null ? ("path="+parent.getPath()) : "");

return newRequest;
}
Expand Down
23 changes: 12 additions & 11 deletions core/libs/image-importer/src/main/java/gallerymine/backend/importer/ImportProcessorBase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ public ImportProcessorBase(ImportApproveRequestPoolManager.StatusHolder statuses
protected boolean validateImportRequest(Process process, Path path) {
if (!path.toFile().exists()) {
String error = request.addError("Path not found for request : %s", path.toFile().getAbsolutePath());
log.error(this.getClass().getSimpleName()+" "+error);
log.error(" "+error);

request.setStatus(statusHolder.getProcessingDone());
requestRepository.save(request);
Expand All @@ -87,7 +87,7 @@ protected boolean validateImportRequest(Process process, Path path) {
public void run() {
Process process = null;
try {
log.info(this.getClass().getSimpleName()+" processing started for {}", request.getPath());
log.info(" processing started for {}", request.getPath());
process = processRepository.findByIdInAndTypeIs(request.getIndexProcessIds(), processType);
if (process == null) {
process = new Process();
Expand All @@ -100,40 +100,40 @@ public void run() {

processRequest(request, process);

log.info(this.getClass().getSimpleName()+" processing started successfuly for {}", request.getPath());
log.info(" processing started successfuly for {}", request.getPath());
} catch (Exception e){
log.error(this.getClass().getSimpleName()+" processing failed for {} Reason: {}", request.getPath(), e.getMessage(), e);
log.error(" processing failed for {} Reason: {}", request.getPath(), e.getMessage(), e);
}
}

protected ImportRequest checkRequest(ImportRequest requestSrc) {
ImportRequest request = requestRepository.findOne(requestSrc.getId());
if (request == null) {
log.info(this.getClass().getSimpleName()+" not found for id={} and path={}", requestSrc.getId(), requestSrc.getPath());
log.info(" not found for id={} and path={}", requestSrc.getId(), requestSrc.getPath());
return null;
}
if (!statusHolder.getAwaitingProcessing().equals(request.getStatus())) {
log.info(this.getClass().getSimpleName()+" status is not processable id={} status={} path={}", request.getId(), request.getStatus(), request.getPath());
log.info(" status is not processable id={} status={} path={}", request.getId(), request.getStatus(), request.getPath());
return null;
}

// request.setStatus(statusHolder.getAwaitingProcessing());
// requestRepository.save(request);
// log.info(this.getClass().getSimpleName()+" status changed id={} status={} path={}", request.getId(), request.getStatus(), request.getPath());
// log.info(" status changed id={} status={} path={}", request.getId(), request.getStatus(), request.getPath());

return request;
}

public abstract void requestProcessing(ImportRequest requestSrc, Process process) throws ImportFailedException;

public void processRequest(ImportRequest requestSrc, Process process) {
log.info(this.getClass().getSimpleName()+" picked up id={} status={} path={}", requestSrc.getId(), requestSrc.getStatus(), requestSrc.getPath());
log.info(" picked up id={} status={} path={}", requestSrc.getId(), requestSrc.getStatus(), requestSrc.getPath());
ImportRequest request = checkRequest(requestSrc);
if (request == null) {
log.info(this.getClass().getSimpleName()+" skipped id={} status={} path={}", requestSrc.getId(), requestSrc.getStatus(), requestSrc.getPath());
log.info(" skipped id={} status={} path={}", requestSrc.getId(), requestSrc.getStatus(), requestSrc.getPath());
return;
}
log.info(this.getClass().getSimpleName()+" started processing id={} status={} path={}", request.getId(), request.getStatus(), request.getPath());
log.info(" processing started id={} status={} path={}", request.getId(), request.getStatus(), request.getPath());

try {
requestProcessing(requestSrc, process);
Expand All @@ -144,8 +144,9 @@ public void processRequest(ImportRequest requestSrc, Process process) {
request.addError(e.getMessage());
requestRepository.save(request);
importService.finishRequestProcessing(request);
log.info(this.getClass().getSimpleName()+" status changed id={} status={} path={}", request.getId(), request.getStatus(), request.getPath());
log.info(" status changed id={} status={} path={}", request.getId(), request.getStatus(), request.getPath());
} finally {
log.info(" processing done id={} status={} path={}", request.getId(), request.getStatus(), request.getPath());
pool.checkForAwaitingRequests();
}
}
Expand Down
Loading

0 comments on commit 223d0b0

Please sign in to comment.