#37 pictureFolder implementation - saving during import

core/beans/src/main/java/gallerymine/model/FileInformation.java

@@ -29,9 +29,15 @@ public class FileInformation implements Comparable<FileInformation> {
 
     @Indexed
     private String storage;
+
+    /** PictureFolder Id*/
+    @Indexed
+    private String folderId;
     @Indexed
     private String filePath;
     @Indexed
+    private String filePathOriginal;
+    @Indexed
     private String fileName;
     @Indexed
     private String fileNameOriginal;
@@ -112,6 +118,12 @@ public String getFileWithPath() {
         return path.toString();
     }
 
+    /** Returns file path without <b>rootPath</b>*/
+    public Path getFileWithPathAsPath() {
+        Path path = Paths.get(filePath, fileName);
+        return path;
+    }
+
     public String getLocation() {
         return
             (StringUtils.isNotBlank(storage) ? (storage+":") : "") +
@@ -125,6 +137,7 @@ public <FI extends FileInformation> void copyFrom(FI sourceToMatch) {
 
         rootPath = sourceToMatch.getRootPath();
         filePath = sourceToMatch.getFilePath();
+        filePathOriginal = sourceToMatch.getFilePathOriginal();
         fileName = sourceToMatch.getFileName();
         fileNameOriginal = sourceToMatch.getFileNameOriginal();
 

core/beans/src/main/java/gallerymine/model/PictureFolder.java

@@ -0,0 +1,78 @@
+package gallerymine.model;
+
+import lombok.Data;
+import org.joda.time.DateTime;
+import org.springframework.data.annotation.CreatedDate;
+import org.springframework.data.annotation.Id;
+import org.springframework.data.annotation.LastModifiedDate;
+import org.springframework.data.annotation.Version;
+import org.springframework.data.mongodb.core.index.Indexed;
+import org.springframework.data.mongodb.core.mapping.Document;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * This bean holds information about request for indexing
+ * Created by sergii_puliaiev on 7/30/18.
+ */
+@Document(collection = "pictureFolder")
+@Data
+public class PictureFolder {
+
+    @Id
+    private String id;
+
+    @Indexed
+    private String name;
+    @Indexed
+    private String namel;
+    @Indexed
+    private String parentId;
+
+    long filesCount;
+    long foldersCount;
+
+    /** Current folder path with name */
+    private String path;
+
+    /** Current folder path with name, lowercased to support all filesystems */
+    @Indexed(unique = true)
+    private String pathl;
+
+    private List<String> notes = new ArrayList<>();
+
+    @CreatedDate
+    private DateTime created;
+    @LastModifiedDate
+    private DateTime updated;
+
+    @Version
+    private Long version;
+
+    public PictureFolder() {
+    }
+
+    public String addNote(String note, Object... params) {
+        if (params!= null && params.length > 0) {
+            note = String.format(note, params);
+        }
+        notes.add(note);
+        return note;
+    }
+
+    public String notesText() {
+        return notes.stream().collect(Collectors.joining("\n"));
+    }
+
+    public void setName(String name) {
+        this.name = name;
+        namel = name == null ? null : name.toLowerCase().replaceAll("^[_$/\\\\]*", "");
+    }
+
+    public void setPath(String path) {
+        this.path = path;
+        pathl = path == null ? null : path.toLowerCase();
+    }
+
+}

core/beans/src/main/java/gallerymine/model/importer/ImportRequest.java

@@ -342,17 +342,20 @@ public String marker() {
         StringBuilder res = new StringBuilder();
         int CUT_PART = 5;
         if (activeProcessId != null && activeProcessId.length() > CUT_PART) {
+            res.append("IP");
             int len = activeProcessId.length();
             res.append(activeProcessId, len-CUT_PART, len);
         }
         res.append(":");
         if (rootId != null && rootId.length() > CUT_PART) {
             int len = rootId.length();
+            res.append("IR");
             res.append(rootId, len-CUT_PART, len);
         }
         res.append(":");
         if (id != null && id.length() > CUT_PART) {
             int len = id.length();
+            res.append("IR");
             res.append(id, len-CUT_PART, len);
         }
         res.append("[");

core/dao/src/main/java/gallerymine/backend/beans/repository/PictureFolderRepository.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2012-2013 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package gallerymine.backend.beans.repository;
+
+import gallerymine.model.PictureFolder;
+import org.springframework.data.mongodb.repository.MongoRepository;
+import org.springframework.data.repository.query.Param;
+import org.springframework.stereotype.Repository;
+
+import java.util.Collection;
+
+@Repository
+public interface PictureFolderRepository extends MongoRepository<PictureFolder, String>, PictureFolderRepositoryCustom {
+
+	Collection<PictureFolder> findByName(@Param("name") String name);
+	Collection<PictureFolder> findByNamel(@Param("namel") String namel);
+
+	PictureFolder findByPath(@Param("path") String path);
+	PictureFolder findByPathl(@Param("pathl") String pathl);
+
+}

core/dao/src/main/java/gallerymine/backend/beans/repository/PictureFolderRepositoryCustom.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2012-2013 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package gallerymine.backend.beans.repository;
+
+import org.springframework.stereotype.Repository;
+
+@Repository
+public interface PictureFolderRepositoryCustom {
+
+	long incrementFilesCount(String picFolderId);
+	long incrementFoldersCount(String picFolderId);
+
+}

core/dao/src/main/java/gallerymine/backend/beans/repository/PictureFolderRepositoryImpl.java

@@ -0,0 +1,43 @@
+package gallerymine.backend.beans.repository;
+
+import gallerymine.backend.data.RetryVersion;
+import gallerymine.model.PictureFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.data.mongodb.core.MongoTemplate;
+import org.springframework.stereotype.Repository;
+
+/**
+ * Source repository with custom methods
+ * Created by sergii_puliaiev on 6/19/17.
+ */
+@Repository
+public class PictureFolderRepositoryImpl implements PictureFolderRepositoryCustom {
+
+    private static Logger log = LoggerFactory.getLogger(PictureFolderRepositoryImpl.class);
+
+    @Autowired
+    protected MongoTemplate template = null;
+
+    @Override
+    @RetryVersion(times = 10, on = org.springframework.dao.OptimisticLockingFailureException.class)
+    public long incrementFilesCount(String picFolderId) {
+        PictureFolder picFolder = template.findById(picFolderId, PictureFolder.class);
+        picFolder.setFilesCount(picFolder.getFilesCount()+1);
+        template.save(picFolder);
+        log.info(" incFiles for folder new={} path={}", picFolder.getFilesCount(), picFolder.getPath());
+        return picFolder.getFilesCount();
+    }
+
+    @Override
+    @RetryVersion(times = 10, on = org.springframework.dao.OptimisticLockingFailureException.class)
+    public long incrementFoldersCount(String picFolderId) {
+        PictureFolder picFolder = template.findById(picFolderId, PictureFolder.class);
+        picFolder.setFoldersCount(picFolder.getFoldersCount()+1);
+        template.save(picFolder);
+        log.info(" incFolders for folder new={} path={}", picFolder.getFilesCount(), picFolder.getPath());
+        return picFolder.getFoldersCount();
+    }
+
+}

core/libs/image-importer/src/main/java/gallerymine/backend/exceptions/ImportFailedException.java

@@ -13,6 +13,27 @@ public ImportFailedException(String message, Throwable cause) {
         super(message, cause);
     }
 
+    public static Throwable findCause(Object ... params) {
+        Throwable cause = null;
+        if (params != null && params.length > 0) {
+            if (params[params.length-1] instanceof Throwable) {
+                cause = (Throwable) params[params.length-1];
+            }
+        }
+        return cause;
+    }
+
+    public static String formatMessage(String message, Object... params) {
+        if (params == null || params.length == 0) {
+            return message;
+        }
+        return String.format(message, params);
+    }
+
+    public ImportFailedException(String message, Object... params) {
+        super(formatMessage(message), findCause(params));
+    }
+
     public ImportFailedException(Throwable cause) {
         super(cause);
     }

core/libs/image-importer/src/main/java/gallerymine/backend/importer/ImportProcessor.java

@@ -199,7 +199,8 @@ public void requestProcessing() throws ImportFailedException {
                     info.setImportRequestRootId(request.getRootId());
 
                     info.setRootPath(request.getRootPath());
-                    info.setFilePath(appConfig.relativizePath(file.getParent(), importRootFullFolder));
+                    info.setFilePathOriginal(appConfig.relativizePath(file.getParent(), importRootFullFolder));
+                    info.setFilePath(info.getFilePathOriginal());
                     info.setFileName(file.toFile().getName());
                     info.setFileNameOriginal(file.toFile().getName());
 

core/libs/image-importer/src/main/java/gallerymine/backend/services/ImportService.java

@@ -3,6 +3,7 @@
 import gallerymine.backend.beans.AppConfig;
 import gallerymine.backend.beans.repository.ImportRequestRepository;
 import gallerymine.backend.beans.repository.ImportSourceRepository;
+import gallerymine.backend.beans.repository.PictureFolderRepository;
 import gallerymine.backend.beans.repository.ProcessRepository;
 import gallerymine.backend.data.RetryVersion;
 import gallerymine.backend.exceptions.ImageApproveException;
@@ -13,9 +14,7 @@
 import gallerymine.backend.pool.ImportMatchingRequestPoolManager;
 import gallerymine.backend.pool.ImportRequestPoolManager;
 import gallerymine.backend.utils.ImportUtils;
-import gallerymine.model.ImportSource;
-import gallerymine.model.Picture;
-import gallerymine.model.PictureInformation;
+import gallerymine.model.*;
 import gallerymine.model.Process;
 import gallerymine.model.importer.ImportRequest;
 import gallerymine.model.mvc.SourceCriteria;
@@ -32,6 +31,7 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.*;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Stream;
@@ -55,6 +55,9 @@ public class ImportService {
     @Autowired
     private ImportUtils importUtils;
 
+    @Autowired
+    private PictureFolderRepository pictureFolderRepository;
+
     @Autowired
     private AppConfig appConfig;
 
@@ -393,16 +396,57 @@ public Path indexateFileIfNeeded(Path file) {
         return file;
     }
 
+    @RetryVersion(times = 10, on = org.springframework.dao.OptimisticLockingFailureException.class)
+    public PictureFolder getOrCreatePictureFolder(Path folder) throws ImportFailedException {
+        try {
+            if (folder == null) {
+                PictureFolder picFolder = pictureFolderRepository.findByPathl("");
+                if (picFolder == null) {
+                    picFolder = new PictureFolder();
+                    picFolder.setName("");
+                    picFolder.setPath("");
+                    pictureFolderRepository.save(picFolder);
+                }
+                return picFolder;
+            }
+            String folderRelPath = folder.toString().toLowerCase();
+            PictureFolder picFolder = pictureFolderRepository.findByPathl(folderRelPath);
+
+            if (picFolder == null) {
+                picFolder = new PictureFolder();
+                picFolder.setName(folder.toFile().getName());
+                picFolder.setPath(folder.toString());
+
+                PictureFolder picFolderParent = getOrCreatePictureFolder(folder.getParent());
+                picFolder.setParentId(picFolderParent.getId());
+
+                pictureFolderRepository.save(picFolder);
+                if (picFolder.getParentId() != null) {
+                    pictureFolderRepository.incrementFoldersCount(picFolder.getParentId());
+                }
+            }
+
+            return picFolder;
+        } catch (Exception e) {
+            log.error("Failed to create PictureFolder for path {}. Reason: {}", folder, e.getMessage(), e);
+            throw new ImportFailedException("Failed to create PictureFolder for path {}. Reason: {}", folder, e.getMessage());
+        }
+    }
+
     public PictureInformation settlePicture(PictureInformation source) throws ImageApproveException {
         try {
             log.info("  source id={} is getting approved", source.getId());
 
             Path importImage = importUtils.calcCompleteFilePath(source);
-            Path galleryImagePath = indexateFileIfNeeded(importUtils.calcCompleteFilePath(GALLERY, source.getFileWithPath()));
+
+            Path relativePathWithFile = source.getFileWithPathAsPath();
+            Path relativePathWithFileLowered = Paths.get(relativePathWithFile.toString().toLowerCase());
+            Path galleryImagePath = indexateFileIfNeeded(importUtils.calcCompleteFilePath(GALLERY, relativePathWithFileLowered.toString()));
 
             galleryImagePath.getParent().toFile().mkdirs();
 
             FileUtils.copyFile(importImage.toFile(), galleryImagePath.toFile(), true);
+            PictureFolder picFolder = getOrCreatePictureFolder(relativePathWithFileLowered.getParent());
 
             Picture picture = uniSourceService.retrySave(source.getId(), Picture.class, pic -> {
                 if (pic == null) {
@@ -413,9 +457,19 @@ public PictureInformation settlePicture(PictureInformation source) throws ImageA
                 pic.setGrade(PictureGrade.GALLERY);
                 pic.addImport(source.getId());
                 pic.setRootPath(null);
+                // update to lowered path
+                if (relativePathWithFileLowered.getParent() != null) {
+                    pic.setFilePath(relativePathWithFileLowered.getParent().toString());
+                } else {
+                    pic.setFilePath("");
+                }
+                // update to the lowered and potentially indexed file name
                 pic.setFileName(galleryImagePath.toFile().getName());
+                pic.setFolderId(picFolder.getId());
                 return pic;
             });
+            pictureFolderRepository.incrementFilesCount(picFolder.getId());
+            log.info("  source id={} is saved to gallery as path={}", source.getId(), relativePathWithFileLowered);
 
             return picture;
         } catch (Exception e) {