Skip to content

Commit

Permalink
Scanner: Endpoint for scan and clean
Browse files Browse the repository at this point in the history
  • Loading branch information
Arthi-chaud committed May 20, 2024
1 parent 7e90569 commit 4e883b1
Show file tree
Hide file tree
Showing 11 changed files with 144 additions and 110 deletions.
2 changes: 2 additions & 0 deletions docker-compose.dev.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ services:
context: ./scanner
dockerfile: Dockerfile.dev
restart: on-failure
ports:
- "8133:8133"
depends_on:
api:
condition: service_healthy
Expand Down
3 changes: 3 additions & 0 deletions docker-compose.prod.yml
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ services:
depends_on:
api:
condition: service_healthy
expose:
- "8133"
volumes:
- ${DATA_DIR}:/videos:ro
- ./scanner.json:/app/scanner.json
Expand Down Expand Up @@ -111,6 +113,7 @@ services:
- FRONT_URL=http://front:3000
- SERVER_URL=http://api:8000
- TRANSCODER_URL=http://transcoder:7666
- SCANNER_URL=http://scanner:8133
volumes:
- ./nginx.conf.template:/etc/nginx/templates/blee.conf.template:ro
transcoder:
Expand Down
3 changes: 3 additions & 0 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ services:
scanner:
build:
context: ./scanner
expose:
- "8133"
depends_on:
api:
condition: service_healthy
Expand Down Expand Up @@ -115,6 +117,7 @@ services:
- FRONT_URL=http://front:3000
- SERVER_URL=http://api:8000
- TRANSCODER_URL=http://transcoder:7666
- SCANNER_URL=http://scanner:8133
volumes:
- ./nginx.conf.template:/etc/nginx/templates/blee.conf.template:ro
transcoder:
Expand Down
3 changes: 3 additions & 0 deletions nginx.conf.template
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ server {
location /transcoder/ {
proxy_pass ${TRANSCODER_URL}/;
}
location /scanner/ {
proxy_pass ${SCANNER_URL}/;
}
location / {
proxy_pass ${FRONT_URL};
}
Expand Down
64 changes: 25 additions & 39 deletions scanner/app/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,11 @@ package main

import (
"os"
"slices"

"github.com/Arthi-chaud/Blee/scanner/pkg"
"github.com/Arthi-chaud/Blee/scanner/pkg/actions"
"github.com/Arthi-chaud/Blee/scanner/pkg/api"
"github.com/Arthi-chaud/Blee/scanner/pkg/config"
wa "github.com/Arthi-chaud/Blee/scanner/pkg/watcher"
"github.com/fsnotify/fsnotify"
"github.com/kpango/glg"
"github.com/labstack/echo/v4"
)

func setupLogger() {
Expand All @@ -25,45 +21,35 @@ func setupLogger() {
func main() {
setupLogger()
c := config.GetConfig()
w, err := fsnotify.NewWatcher()
if err != nil {
glg.Fatalf("Failed setting up API: %s", err)
os.Exit(1)
}

if err := api.HealthCheck(c); err != nil {
glg.Fatalf("Failed connecting to API: %s", err)
os.Exit(1)
}
go func() {
for {
select {
case event := <-w.Events:
wa.HandleWatcherEvent(&event, &c)
case err := <-w.Errors:
glg.Fatalf("File System Watcher errored: %s", err)
os.Exit(1)
}
}
}()
glg.Logf("Attempting to watch %s", c.WatchDir)
e := echo.New()

watchedFiles := wa.SetupWatcher(w, c.WatchDir)
knownPaths, err := api.GetAllKnownPaths(c)
if err != nil {
glg.Fatalf("Could not get registered files from API: %s", err)
os.Exit(1)
}
for _, path := range watchedFiles {
if !slices.Contains(knownPaths, path) && pkg.FileIsVideo(path) {
actions.RegisterFile(path, &c)
}
s := ScannerContext {
config: &c,
}
for _, path := range knownPaths {
if !slices.Contains(watchedFiles, path) && pkg.FileIsVideo(path) {
actions.DeleteFile(path, &c)
}
}
glg.Log("Scanner started! Let's get this show on the road.")

<-make(chan struct{})
e.POST("/scan", s.Scan)
e.POST("/clean", s.Clean)

// knownPaths, err := api.GetAllKnownPaths(c)
// if err != nil {
// glg.Fatalf("Could not get registered files from API: %s", err)
// os.Exit(1)
// }
// for _, path := range watchedFiles {
// if !slices.Contains(knownPaths, path) && pkg.FileIsVideo(path) {
// actions.RegisterFile(path, &c)
// }
// }
// for _, path := range knownPaths {
// if !slices.Contains(watchedFiles, path) && pkg.FileIsVideo(path) {
// actions.DeleteFile(path, &c)
// }
// }
glg.Log("Scanner started! Let's get this show on the road.")
e.Logger.Fatal(e.Start(":8133"))
}
55 changes: 55 additions & 0 deletions scanner/app/routes.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
package main

import (
"net/http"
"slices"

"github.com/Arthi-chaud/Blee/scanner/pkg"
"github.com/Arthi-chaud/Blee/scanner/pkg/actions"
"github.com/Arthi-chaud/Blee/scanner/pkg/api"
"github.com/Arthi-chaud/Blee/scanner/pkg/config"
"github.com/labstack/echo/v4"
)

type ScannerContext struct {
config *config.Config
}

func getFilesFromAPIOrFail(s *ScannerContext) ([]string, error) {
knownPaths, err := api.GetAllKnownPaths(*s.config)
if err != nil {
return nil, echo.NewHTTPError(http.StatusServiceUnavailable, "Could not get info from API.")
}
return knownPaths, nil
}

func (s *ScannerContext) Scan(c echo.Context) error {
knownPaths, err := getFilesFromAPIOrFail(s)
if err != nil {
return err
}
go func() {
for _, path := range pkg.GetWatchedFiles(s.config.WatchDir) {
if !slices.Contains(knownPaths, path) && pkg.FileIsVideo(path) {
actions.RegisterFile(path, s.config)
}
}
}()
return nil
}

func (s *ScannerContext) Clean(c echo.Context) error {
knownPaths, err := getFilesFromAPIOrFail(s)
if err != nil {
return err
}
go func() {
watchedFiles := pkg.GetWatchedFiles(s.config.WatchDir)
for _, path := range knownPaths {
if !slices.Contains(watchedFiles, path) && pkg.FileIsVideo(path) {
actions.DeleteFile(path, s.config)
}
}
}()
return nil
}
6 changes: 6 additions & 0 deletions scanner/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,14 @@ require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/labstack/echo/v4 v4.12.0 // indirect
github.com/labstack/gommon v0.4.2 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/u2takey/go-utils v0.3.1 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasttemplate v1.2.2 // indirect
golang.org/x/crypto v0.23.0 // indirect
golang.org/x/net v0.25.0 // indirect
golang.org/x/sys v0.20.0 // indirect
Expand Down
15 changes: 15 additions & 0 deletions scanner/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,17 @@ github.com/kpango/fastime v1.1.9 h1:xVQHcqyPt5M69DyFH7g1EPRns1YQNap9d5eLhl/Jy84=
github.com/kpango/fastime v1.1.9/go.mod h1:vyD7FnUn08zxY4b/QFBZVG+9EWMYsNl+QF0uE46urD4=
github.com/kpango/glg v1.6.15 h1:nw0xSxpSyrDIWHeb3dvnE08PW+SCbK+aYFETT75IeLA=
github.com/kpango/glg v1.6.15/go.mod h1:cmsc7Yeu8AS3wHLmN7bhwENXOpxfq+QoqxCIk2FneRk=
github.com/labstack/echo/v4 v4.12.0 h1:IKpw49IMryVB2p1a4dzwlhP1O2Tf2E0Ir/450lH+kI0=
github.com/labstack/echo/v4 v4.12.0/go.mod h1:UP9Cr2DJXbOK3Kr9ONYzNowSh7HP0aG0ShAyycHSJvM=
github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0=
github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU=
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
Expand All @@ -60,6 +69,10 @@ github.com/u2takey/ffmpeg-go v0.5.0 h1:r7d86XuL7uLWJ5mzSeQ03uvjfIhiJYvsRAJFCW4uk
github.com/u2takey/ffmpeg-go v0.5.0/go.mod h1:ruZWkvC1FEiUNjmROowOAps3ZcWxEiOpFoHCvk97kGc=
github.com/u2takey/go-utils v0.3.1 h1:TaQTgmEZZeDHQFYfd+AdUT1cT4QJgJn/XVPELhHw4ys=
github.com/u2takey/go-utils v0.3.1/go.mod h1:6e+v5vEZ/6gu12w/DC2ixZdZtCrNokVxD0JUklcqdCs=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
github.com/zoriya/go-mediainfo v0.0.0-20240113011752-07018f07efae h1:Xmtcb/GrG1jcLxC3cDFwdAM1oV7fLNBcN3h3YL+gN6g=
github.com/zoriya/go-mediainfo v0.0.0-20240113011752-07018f07efae/go.mod h1:jzun1oQGoJSh65g1XKaolTmjd6HW/34WHH7VMdJdbvM=
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
Expand All @@ -82,6 +95,8 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
Expand Down
32 changes: 32 additions & 0 deletions scanner/pkg/filesystem.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
// Get All Files in watched directory
package pkg

import (
"os"
"path"

"github.com/kpango/glg"
)
func GetWatchedFiles(rootDir string) []string {
watchedFiles := []string{}
entries, err := os.ReadDir(rootDir)

if err != nil {
glg.Fatal(err)
}

for _, e := range entries {
entryName := path.Join(rootDir, e.Name())
i, err := os.Stat(entryName)
if err != nil {
glg.Fail(err)
return []string{}
}
if i.IsDir() {
watchedFiles = append(watchedFiles, GetWatchedFiles(entryName)...)
} else {
watchedFiles = append(watchedFiles, entryName)
}
}
return watchedFiles
}
41 changes: 0 additions & 41 deletions scanner/pkg/watcher/handle_event.go

This file was deleted.

30 changes: 0 additions & 30 deletions scanner/pkg/watcher/watcher.go

This file was deleted.

0 comments on commit 4e883b1

Please sign in to comment.