diff --git a/core/beans/src/main/java/gallerymine/model/FileInformation.java b/core/beans/src/main/java/gallerymine/model/FileInformation.java index 8aa8a13..62f6d6b 100644 --- a/core/beans/src/main/java/gallerymine/model/FileInformation.java +++ b/core/beans/src/main/java/gallerymine/model/FileInformation.java @@ -29,9 +29,15 @@ public class FileInformation implements Comparable { @Indexed private String storage; + + /** PictureFolder Id*/ + @Indexed + private String folderId; @Indexed private String filePath; @Indexed + private String filePathOriginal; + @Indexed private String fileName; @Indexed private String fileNameOriginal; @@ -112,6 +118,12 @@ public String getFileWithPath() { return path.toString(); } + /** Returns file path without rootPath*/ + public Path getFileWithPathAsPath() { + Path path = Paths.get(filePath, fileName); + return path; + } + public String getLocation() { return (StringUtils.isNotBlank(storage) ? (storage+":") : "") + @@ -125,6 +137,7 @@ public void copyFrom(FI sourceToMatch) { rootPath = sourceToMatch.getRootPath(); filePath = sourceToMatch.getFilePath(); + filePathOriginal = sourceToMatch.getFilePathOriginal(); fileName = sourceToMatch.getFileName(); fileNameOriginal = sourceToMatch.getFileNameOriginal(); diff --git a/core/beans/src/main/java/gallerymine/model/PictureFolder.java b/core/beans/src/main/java/gallerymine/model/PictureFolder.java new file mode 100644 index 0000000..e9a3423 --- /dev/null +++ b/core/beans/src/main/java/gallerymine/model/PictureFolder.java @@ -0,0 +1,78 @@ +package gallerymine.model; + +import lombok.Data; +import org.joda.time.DateTime; +import org.springframework.data.annotation.CreatedDate; +import org.springframework.data.annotation.Id; +import org.springframework.data.annotation.LastModifiedDate; +import org.springframework.data.annotation.Version; +import org.springframework.data.mongodb.core.index.Indexed; +import org.springframework.data.mongodb.core.mapping.Document; + +import java.util.*; +import java.util.stream.Collectors; + +/** + * This bean holds information about request for indexing + * Created by sergii_puliaiev on 7/30/18. + */ +@Document(collection = "pictureFolder") +@Data +public class PictureFolder { + + @Id + private String id; + + @Indexed + private String name; + @Indexed + private String namel; + @Indexed + private String parentId; + + long filesCount; + long foldersCount; + + /** Current folder path with name */ + private String path; + + /** Current folder path with name, lowercased to support all filesystems */ + @Indexed(unique = true) + private String pathl; + + private List notes = new ArrayList<>(); + + @CreatedDate + private DateTime created; + @LastModifiedDate + private DateTime updated; + + @Version + private Long version; + + public PictureFolder() { + } + + public String addNote(String note, Object... params) { + if (params!= null && params.length > 0) { + note = String.format(note, params); + } + notes.add(note); + return note; + } + + public String notesText() { + return notes.stream().collect(Collectors.joining("\n")); + } + + public void setName(String name) { + this.name = name; + namel = name == null ? null : name.toLowerCase().replaceAll("^[_$/\\\\]*", ""); + } + + public void setPath(String path) { + this.path = path; + pathl = path == null ? null : path.toLowerCase(); + } + +} diff --git a/core/beans/src/main/java/gallerymine/model/importer/ImportRequest.java b/core/beans/src/main/java/gallerymine/model/importer/ImportRequest.java index 3764f9e..b0ab8f1 100644 --- a/core/beans/src/main/java/gallerymine/model/importer/ImportRequest.java +++ b/core/beans/src/main/java/gallerymine/model/importer/ImportRequest.java @@ -342,17 +342,20 @@ public String marker() { StringBuilder res = new StringBuilder(); int CUT_PART = 5; if (activeProcessId != null && activeProcessId.length() > CUT_PART) { + res.append("IP"); int len = activeProcessId.length(); res.append(activeProcessId, len-CUT_PART, len); } res.append(":"); if (rootId != null && rootId.length() > CUT_PART) { int len = rootId.length(); + res.append("IR"); res.append(rootId, len-CUT_PART, len); } res.append(":"); if (id != null && id.length() > CUT_PART) { int len = id.length(); + res.append("IR"); res.append(id, len-CUT_PART, len); } res.append("["); diff --git a/core/dao/src/main/java/gallerymine/backend/beans/repository/PictureFolderRepository.java b/core/dao/src/main/java/gallerymine/backend/beans/repository/PictureFolderRepository.java new file mode 100644 index 0000000..91c04e3 --- /dev/null +++ b/core/dao/src/main/java/gallerymine/backend/beans/repository/PictureFolderRepository.java @@ -0,0 +1,35 @@ +/* + * Copyright 2012-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package gallerymine.backend.beans.repository; + +import gallerymine.model.PictureFolder; +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; + +import java.util.Collection; + +@Repository +public interface PictureFolderRepository extends MongoRepository, PictureFolderRepositoryCustom { + + Collection findByName(@Param("name") String name); + Collection findByNamel(@Param("namel") String namel); + + PictureFolder findByPath(@Param("path") String path); + PictureFolder findByPathl(@Param("pathl") String pathl); + +} diff --git a/core/dao/src/main/java/gallerymine/backend/beans/repository/PictureFolderRepositoryCustom.java b/core/dao/src/main/java/gallerymine/backend/beans/repository/PictureFolderRepositoryCustom.java new file mode 100644 index 0000000..14a9a57 --- /dev/null +++ b/core/dao/src/main/java/gallerymine/backend/beans/repository/PictureFolderRepositoryCustom.java @@ -0,0 +1,27 @@ +/* + * Copyright 2012-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package gallerymine.backend.beans.repository; + +import org.springframework.stereotype.Repository; + +@Repository +public interface PictureFolderRepositoryCustom { + + long incrementFilesCount(String picFolderId); + long incrementFoldersCount(String picFolderId); + +} diff --git a/core/dao/src/main/java/gallerymine/backend/beans/repository/PictureFolderRepositoryImpl.java b/core/dao/src/main/java/gallerymine/backend/beans/repository/PictureFolderRepositoryImpl.java new file mode 100644 index 0000000..bd1b07c --- /dev/null +++ b/core/dao/src/main/java/gallerymine/backend/beans/repository/PictureFolderRepositoryImpl.java @@ -0,0 +1,43 @@ +package gallerymine.backend.beans.repository; + +import gallerymine.backend.data.RetryVersion; +import gallerymine.model.PictureFolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.stereotype.Repository; + +/** + * Source repository with custom methods + * Created by sergii_puliaiev on 6/19/17. + */ +@Repository +public class PictureFolderRepositoryImpl implements PictureFolderRepositoryCustom { + + private static Logger log = LoggerFactory.getLogger(PictureFolderRepositoryImpl.class); + + @Autowired + protected MongoTemplate template = null; + + @Override + @RetryVersion(times = 10, on = org.springframework.dao.OptimisticLockingFailureException.class) + public long incrementFilesCount(String picFolderId) { + PictureFolder picFolder = template.findById(picFolderId, PictureFolder.class); + picFolder.setFilesCount(picFolder.getFilesCount()+1); + template.save(picFolder); + log.info(" incFiles for folder new={} path={}", picFolder.getFilesCount(), picFolder.getPath()); + return picFolder.getFilesCount(); + } + + @Override + @RetryVersion(times = 10, on = org.springframework.dao.OptimisticLockingFailureException.class) + public long incrementFoldersCount(String picFolderId) { + PictureFolder picFolder = template.findById(picFolderId, PictureFolder.class); + picFolder.setFoldersCount(picFolder.getFoldersCount()+1); + template.save(picFolder); + log.info(" incFolders for folder new={} path={}", picFolder.getFilesCount(), picFolder.getPath()); + return picFolder.getFoldersCount(); + } + +} diff --git a/core/libs/image-importer/src/main/java/gallerymine/backend/exceptions/ImportFailedException.java b/core/libs/image-importer/src/main/java/gallerymine/backend/exceptions/ImportFailedException.java index df1d22c..7a38cc7 100644 --- a/core/libs/image-importer/src/main/java/gallerymine/backend/exceptions/ImportFailedException.java +++ b/core/libs/image-importer/src/main/java/gallerymine/backend/exceptions/ImportFailedException.java @@ -13,6 +13,27 @@ public ImportFailedException(String message, Throwable cause) { super(message, cause); } + public static Throwable findCause(Object ... params) { + Throwable cause = null; + if (params != null && params.length > 0) { + if (params[params.length-1] instanceof Throwable) { + cause = (Throwable) params[params.length-1]; + } + } + return cause; + } + + public static String formatMessage(String message, Object... params) { + if (params == null || params.length == 0) { + return message; + } + return String.format(message, params); + } + + public ImportFailedException(String message, Object... params) { + super(formatMessage(message), findCause(params)); + } + public ImportFailedException(Throwable cause) { super(cause); } diff --git a/core/libs/image-importer/src/main/java/gallerymine/backend/importer/ImportProcessor.java b/core/libs/image-importer/src/main/java/gallerymine/backend/importer/ImportProcessor.java index 3e59720..043af1e 100644 --- a/core/libs/image-importer/src/main/java/gallerymine/backend/importer/ImportProcessor.java +++ b/core/libs/image-importer/src/main/java/gallerymine/backend/importer/ImportProcessor.java @@ -199,7 +199,8 @@ public void requestProcessing() throws ImportFailedException { info.setImportRequestRootId(request.getRootId()); info.setRootPath(request.getRootPath()); - info.setFilePath(appConfig.relativizePath(file.getParent(), importRootFullFolder)); + info.setFilePathOriginal(appConfig.relativizePath(file.getParent(), importRootFullFolder)); + info.setFilePath(info.getFilePathOriginal()); info.setFileName(file.toFile().getName()); info.setFileNameOriginal(file.toFile().getName()); diff --git a/core/libs/image-importer/src/main/java/gallerymine/backend/services/ImportService.java b/core/libs/image-importer/src/main/java/gallerymine/backend/services/ImportService.java index 1219970..a2f4fe6 100644 --- a/core/libs/image-importer/src/main/java/gallerymine/backend/services/ImportService.java +++ b/core/libs/image-importer/src/main/java/gallerymine/backend/services/ImportService.java @@ -3,6 +3,7 @@ import gallerymine.backend.beans.AppConfig; import gallerymine.backend.beans.repository.ImportRequestRepository; import gallerymine.backend.beans.repository.ImportSourceRepository; +import gallerymine.backend.beans.repository.PictureFolderRepository; import gallerymine.backend.beans.repository.ProcessRepository; import gallerymine.backend.data.RetryVersion; import gallerymine.backend.exceptions.ImageApproveException; @@ -13,9 +14,7 @@ import gallerymine.backend.pool.ImportMatchingRequestPoolManager; import gallerymine.backend.pool.ImportRequestPoolManager; import gallerymine.backend.utils.ImportUtils; -import gallerymine.model.ImportSource; -import gallerymine.model.Picture; -import gallerymine.model.PictureInformation; +import gallerymine.model.*; import gallerymine.model.Process; import gallerymine.model.importer.ImportRequest; import gallerymine.model.mvc.SourceCriteria; @@ -32,6 +31,7 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.*; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Stream; @@ -55,6 +55,9 @@ public class ImportService { @Autowired private ImportUtils importUtils; + @Autowired + private PictureFolderRepository pictureFolderRepository; + @Autowired private AppConfig appConfig; @@ -393,16 +396,57 @@ public Path indexateFileIfNeeded(Path file) { return file; } + @RetryVersion(times = 10, on = org.springframework.dao.OptimisticLockingFailureException.class) + public PictureFolder getOrCreatePictureFolder(Path folder) throws ImportFailedException { + try { + if (folder == null) { + PictureFolder picFolder = pictureFolderRepository.findByPathl(""); + if (picFolder == null) { + picFolder = new PictureFolder(); + picFolder.setName(""); + picFolder.setPath(""); + pictureFolderRepository.save(picFolder); + } + return picFolder; + } + String folderRelPath = folder.toString().toLowerCase(); + PictureFolder picFolder = pictureFolderRepository.findByPathl(folderRelPath); + + if (picFolder == null) { + picFolder = new PictureFolder(); + picFolder.setName(folder.toFile().getName()); + picFolder.setPath(folder.toString()); + + PictureFolder picFolderParent = getOrCreatePictureFolder(folder.getParent()); + picFolder.setParentId(picFolderParent.getId()); + + pictureFolderRepository.save(picFolder); + if (picFolder.getParentId() != null) { + pictureFolderRepository.incrementFoldersCount(picFolder.getParentId()); + } + } + + return picFolder; + } catch (Exception e) { + log.error("Failed to create PictureFolder for path {}. Reason: {}", folder, e.getMessage(), e); + throw new ImportFailedException("Failed to create PictureFolder for path {}. Reason: {}", folder, e.getMessage()); + } + } + public PictureInformation settlePicture(PictureInformation source) throws ImageApproveException { try { log.info(" source id={} is getting approved", source.getId()); Path importImage = importUtils.calcCompleteFilePath(source); - Path galleryImagePath = indexateFileIfNeeded(importUtils.calcCompleteFilePath(GALLERY, source.getFileWithPath())); + + Path relativePathWithFile = source.getFileWithPathAsPath(); + Path relativePathWithFileLowered = Paths.get(relativePathWithFile.toString().toLowerCase()); + Path galleryImagePath = indexateFileIfNeeded(importUtils.calcCompleteFilePath(GALLERY, relativePathWithFileLowered.toString())); galleryImagePath.getParent().toFile().mkdirs(); FileUtils.copyFile(importImage.toFile(), galleryImagePath.toFile(), true); + PictureFolder picFolder = getOrCreatePictureFolder(relativePathWithFileLowered.getParent()); Picture picture = uniSourceService.retrySave(source.getId(), Picture.class, pic -> { if (pic == null) { @@ -413,9 +457,19 @@ public PictureInformation settlePicture(PictureInformation source) throws ImageA pic.setGrade(PictureGrade.GALLERY); pic.addImport(source.getId()); pic.setRootPath(null); + // update to lowered path + if (relativePathWithFileLowered.getParent() != null) { + pic.setFilePath(relativePathWithFileLowered.getParent().toString()); + } else { + pic.setFilePath(""); + } + // update to the lowered and potentially indexed file name pic.setFileName(galleryImagePath.toFile().getName()); + pic.setFolderId(picFolder.getId()); return pic; }); + pictureFolderRepository.incrementFilesCount(picFolder.getId()); + log.info(" source id={} is saved to gallery as path={}", source.getId(), relativePathWithFileLowered); return picture; } catch (Exception e) { diff --git a/docs/database/cleanup.js b/docs/database/cleanup.js index a1a0be9..a8c7026 100644 --- a/docs/database/cleanup.js +++ b/docs/database/cleanup.js @@ -1,4 +1,4 @@ show dbs; use galleryMineReal3; show collections; -db.importRequest.drop(); db.importSource.drop(); db.indexRequest.drop(); db.picture.drop(); db.pictureInformation.drop(); db.process.drop(); db.source.drop(); db.thumbRequest.drop(); +db.pictureFolder.drop();db.importRequest.drop(); db.importSource.drop(); db.indexRequest.drop(); db.picture.drop(); db.pictureInformation.drop(); db.process.drop(); db.source.drop(); db.thumbRequest.drop(); diff --git a/ui/web/Dockerfile b/ui/web/Dockerfile index 8bfaa51..3b709f0 100644 --- a/ui/web/Dockerfile +++ b/ui/web/Dockerfile @@ -1,5 +1,5 @@ FROM openjdk -MAINTAINER Piotr Minkowski -ADD target/customer-service.jar customer-service.jar -ENTRYPOINT ["java", "-jar", "/customer-service.jar"] +MAINTAINER Sergii Puliaiev +#ADD target/customer-service.jar customer-service.jar +#ENTRYPOINT ["java", "-jar", "/customer-service.jar"] EXPOSE 3333 \ No newline at end of file diff --git a/ui/web/src/main/resources/application.yml b/ui/web/src/main/resources/application.yml new file mode 100644 index 0000000..cf4db92 --- /dev/null +++ b/ui/web/src/main/resources/application.yml @@ -0,0 +1,71 @@ +spring: + application: + name: GalleryMine + data: + rest.base-path: /api + mongodb.uri: mongodb://localhost:27017/galleryMineReal3 + + # Allow Thymeleaf templates to be reloaded at dev time + thymeleaf.cache: false + jmx.default-domain: gallerymine + +# Set whether to expose the Spring-managed Scheduler instance in the Quartz SchedulerRepository. +quartz.scheduler-factory.expose-scheduler-in-repository: false + +server: + address: 127.0.0.1 + port: 8070 + tomcat: + access_log_enabled: true + basedir: target/tomcat +# SSL.key-store: +# servletpath: +# context-path: + error.include-stacktrace: always + +logging: + pattern.level: "%-27.27X{marker}%5p" + level.: warn + level: + org.springframework.data=debug + org.springframework.web.servlet.resource.PathResourceResolver=trace + org.springframework.web.servlet.resource.AbstractResourceResolver=trace + + +application.api.google: + geocode=AIzaSyBkyXKYbKdEpdJhlnUcypn9ZVI5gOodBNU + serverName=GalleryMineServer-Mac + +gallery: + paths: + galleryRootFolder: ~/work_mine/data/gallery/ + sourcesRootFolder: ~/work_mine/data/rootFolder/ + thumbsRootFolder: ~/work_mine/data/thumbsFolder/ + importRootFolder: ~/work_mine/data/importFolder/ + importExposedRootFolder: ~/work_mine/data/importExposedFolder/ + + debug: + dryRunImportMoves: true + import: + abandoned_timeout_ms: 1800000 + disableThumbs: false + + + +endpoints.jmx_enabled: true +endpoints.HEALTH.enabled: true + +# JavaMelody configuration: +# https://github.com/javamelody/javamelody/wiki/SpringBootStarter +management: + security.enabled: false + endpoints.web.exposure.include: "*" + metrics: + export.datadog.enabled: false + use-global-registry: false + +javamelody: + # Enable JavaMelody auto-configuration (optional, default: true) + enabled: true + quartz-default-listener-disabled: false +