fix: made Google Sheets metadata source behave similarly to Yandex Di…

…sk source
yurii-litvinov · Oct 18, 2024 · 60b471d · 60b471d
1 parent 8b4cb01
commit 60b471d
Show file tree

Hide file tree

Showing 4 changed files with 58 additions and 51 deletions.
diff --git a/src/ADP2.CLI/Program.fs b/src/ADP2.CLI/Program.fs
@@ -26,7 +26,7 @@ let main _ =
             printfn "Works with no files:"
 
             knowledgeBase.WorksWithNoFiles
-            |> Seq.iter (fun w -> printfn "%s: %s" w.ShortName w.Title)
+            |> Seq.iter (fun w -> printfn "%s: (transliterated as '%s') %s" w.ShortName (DataModelUtils.transliterate w.ShortName) w.Title)
 
             printfn ""
             printfn "Works with no metainformation:"

diff --git a/src/ADP2.Core/MetadataSources/GoogleSheetsMetadataSource.fs b/src/ADP2.Core/MetadataSources/GoogleSheetsMetadataSource.fs
@@ -14,18 +14,9 @@ type GoogleSheetsConfig = { GoogleSheetId: string }
 type GoogleSheetsMetadataSource(appConfig: ApplicationConfig) =
 
     let readSheetAsync (config: DataConfig) (sheet: Sheet) =
-        let columns =
-            [ config.AuthorNameColumn
-              config.AdvisorColumn
-              config.TitleColumn
-              config.ResultColumn
-              config.DoNotPublishColumn ]
-
-        let columns =
-            if config.SourceUriColumn = "-" then
-                columns
-            else
-                config.CommitterNameColumn :: config.SourceUriColumn :: columns
+        let columns = getInterestingColumns config
+
+        let sourceUriDefined, doNotPublishDefined = isDefinedOptionalColumns config
 
         task {
             let! rows = sheet.ReadByHeadersAsync(columns)
@@ -34,24 +25,28 @@ type GoogleSheetsMetadataSource(appConfig: ApplicationConfig) =
                 rows
                 |> Seq.choose (fun row ->
                     if allowedResults.Contains row[config.ResultColumn] then
-                        if config.SourceUriColumn = "-" then
-                            Some
-                            <| createWorkMetadata
-                                row[config.AuthorNameColumn]
-                                row[config.AdvisorColumn]
-                                row[config.TitleColumn]
-                                ""
+
+                        let sourceUri = if sourceUriDefined then row[config.SourceUriColumn] else ""
+
+                        let commiterName =
+                            if sourceUriDefined && sourceUri <> "" && sourceUri <> "—" then
+                                row[config.CommitterNameColumn]
+                            else
                                 ""
+
+                        let doNotPublish =
+                            if doNotPublishDefined then
                                 row[config.DoNotPublishColumn]
-                        else
-                            Some
-                            <| createWorkMetadata
+                            else
+                                ""
+
+                        Some (createWorkMetadata
                                 row[config.AuthorNameColumn]
                                 row[config.AdvisorColumn]
                                 row[config.TitleColumn]
-                                row[config.SourceUriColumn]
-                                row[config.CommitterNameColumn]
-                                row[config.DoNotPublishColumn]
+                                sourceUri
+                                commiterName
+                                doNotPublish)
                     else
                         None)
         }

diff --git a/src/ADP2.Core/MetadataSources/MetadataSourceUtils.fs b/src/ADP2.Core/MetadataSources/MetadataSourceUtils.fs
@@ -51,6 +51,40 @@ module MetadataSourceUtils =
 
             w)
 
+    /// Gets what optional column names are defined in config
+    let isDefinedOptionalColumns (config: DataConfig) =
+        let sourceUriDefined =
+            not (config.SourceUriColumn = "-" || config.SourceUriColumn = "")
+
+        let doNotPublishDefined =
+            not (config.DoNotPublishColumn = "-" || config.DoNotPublishColumn = "")
+
+        (sourceUriDefined, doNotPublishDefined)
+
+    /// Gets "interesting" column names from config, ignoring undefined optional columns
+    let getInterestingColumns (config: DataConfig) =
+        let columns =
+            [ config.AuthorNameColumn
+              config.AdvisorColumn
+              config.TitleColumn
+              config.ResultColumn ]
+
+        let sourceUriDefined, doNotPublishDefined = isDefinedOptionalColumns config
+
+        let columns =
+            if sourceUriDefined then
+                config.CommitterNameColumn :: config.SourceUriColumn :: columns
+            else
+                columns
+
+        let columns =
+            if doNotPublishDefined then
+                config.DoNotPublishColumn :: columns
+            else
+                columns
+
+        columns
+
     /// Constructs new Work record.
     let createWorkMetadata author advisor title (sourceUrl: string) committerName doNotPublish =
         let cleanup (str: string) =
@@ -59,9 +93,7 @@ module MetadataSourceUtils =
         let work = Work(author |> getSurname)
         let advisor = parseAdvisor advisor
         work.AdvisorName <- fst advisor
-
-        /// Kosarev is the special case, since he is the only one that registered on a site in english.
-        work.AdvisorSurname <- if snd advisor = "Косарев" then "Kosarev" else snd advisor
+        work.AdvisorSurname <- snd advisor
         work.AuthorName <- author
         work.Title <- cleanup title
         work.SourceUri <- if sourceUrl.StartsWith "http" then sourceUrl else ""

diff --git a/src/ADP2.Core/MetadataSources/YandexDiskMetadataSource.fs b/src/ADP2.Core/MetadataSources/YandexDiskMetadataSource.fs
@@ -17,29 +17,9 @@ type YandexSheetConfig =
 type YandexDiskMetadataSource(appConfig: ApplicationConfig) =
 
     let readSheet (config: DataConfig) (sheet: DocUtils.Xlsx.Sheet) =
-        let columns =
-            [ config.AuthorNameColumn
-              config.AdvisorColumn
-              config.TitleColumn
-              config.ResultColumn ]
+        let columns = getInterestingColumns config
 
-        let sourceUriDefined =
-            not (config.SourceUriColumn = "-" || config.SourceUriColumn = "")
-
-        let doNotPublishDefined =
-            not (config.DoNotPublishColumn = "-" || config.DoNotPublishColumn = "")
-
-        let columns =
-            if sourceUriDefined then
-                config.CommitterNameColumn :: config.SourceUriColumn :: columns
-            else
-                columns
-
-        let columns =
-            if doNotPublishDefined then
-                config.DoNotPublishColumn :: columns
-            else
-                columns
+        let sourceUriDefined, doNotPublishDefined = isDefinedOptionalColumns config
 
         let createMetadata (row: Map<string, string>) =
             let sourceUri = if sourceUriDefined then row[config.SourceUriColumn] else ""