From 305c058f4d3636018245f48649d74a054e784839 Mon Sep 17 00:00:00 2001 From: Infinite Date: Mon, 14 Dec 2020 00:09:02 +0000 Subject: [PATCH 1/5] Fix integer overflow for scene size on 32bit systems --- graphql/schema/types/stats.graphql | 4 ++-- pkg/api/resolver.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/graphql/schema/types/stats.graphql b/graphql/schema/types/stats.graphql index 2ce9b6a7674..d127871f2cb 100644 --- a/graphql/schema/types/stats.graphql +++ b/graphql/schema/types/stats.graphql @@ -1,8 +1,8 @@ type StatsResultType { scene_count: Int! - scenes_size: Int! + scenes_size: Float! image_count: Int! - images_size: Int! + images_size: Float! gallery_count: Int! performer_count: Int! studio_count: Int! diff --git a/pkg/api/resolver.go b/pkg/api/resolver.go index 0ef308a336b..c19ba26c6ef 100644 --- a/pkg/api/resolver.go +++ b/pkg/api/resolver.go @@ -133,9 +133,9 @@ func (r *queryResolver) Stats(ctx context.Context) (*models.StatsResultType, err tagsCount, _ := tagsQB.Count() return &models.StatsResultType{ SceneCount: scenesCount, - ScenesSize: int(scenesSize), + ScenesSize: float64(scenesSize), ImageCount: imageCount, - ImagesSize: int(imageSize), + ImagesSize: float64(imageSize), GalleryCount: galleryCount, PerformerCount: performersCount, StudioCount: studiosCount, From 1c8eeff3ba04fc3ceb0dbeb4356168487244e545 Mon Sep 17 00:00:00 2001 From: Infinite Date: Mon, 14 Dec 2020 08:26:34 +0000 Subject: [PATCH 2/5] Cast to double in sqlite to prevent potential overflow --- pkg/api/resolver.go | 4 ++-- pkg/models/querybuilder_image.go | 4 ++-- pkg/models/querybuilder_scene.go | 4 ++-- pkg/models/querybuilder_sql.go | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/api/resolver.go b/pkg/api/resolver.go index c19ba26c6ef..f9a5237a390 100644 --- a/pkg/api/resolver.go +++ b/pkg/api/resolver.go @@ -133,9 +133,9 @@ func (r *queryResolver) Stats(ctx context.Context) (*models.StatsResultType, err tagsCount, _ := tagsQB.Count() return &models.StatsResultType{ SceneCount: scenesCount, - ScenesSize: float64(scenesSize), + ScenesSize: scenesSize, ImageCount: imageCount, - ImagesSize: float64(imageSize), + ImagesSize: imageSize, GalleryCount: galleryCount, PerformerCount: performersCount, StudioCount: studiosCount, diff --git a/pkg/models/querybuilder_image.go b/pkg/models/querybuilder_image.go index 57aebf60703..48ad119b8b4 100644 --- a/pkg/models/querybuilder_image.go +++ b/pkg/models/querybuilder_image.go @@ -246,8 +246,8 @@ func (qb *ImageQueryBuilder) Count() (int, error) { return runCountQuery(buildCountQuery("SELECT images.id FROM images"), nil) } -func (qb *ImageQueryBuilder) Size() (uint64, error) { - return runSumQuery("SELECT SUM(size) as sum FROM images", nil) +func (qb *ImageQueryBuilder) Size() (float64, error) { + return runSumQuery("SELECT SUM(cast(size as double)) as sum FROM images", nil) } func (qb *ImageQueryBuilder) CountByStudioID(studioID int) (int, error) { diff --git a/pkg/models/querybuilder_scene.go b/pkg/models/querybuilder_scene.go index 29ead2e893b..039bb15970a 100644 --- a/pkg/models/querybuilder_scene.go +++ b/pkg/models/querybuilder_scene.go @@ -254,8 +254,8 @@ func (qb *SceneQueryBuilder) Count() (int, error) { return runCountQuery(buildCountQuery("SELECT scenes.id FROM scenes"), nil) } -func (qb *SceneQueryBuilder) Size() (uint64, error) { - return runSumQuery("SELECT SUM(size) as sum FROM scenes", nil) +func (qb *SceneQueryBuilder) Size() (float64, error) { + return runSumQuery("SELECT SUM(cast(size as double)) as sum FROM scenes", nil) } func (qb *SceneQueryBuilder) CountByStudioID(studioID int) (int, error) { diff --git a/pkg/models/querybuilder_sql.go b/pkg/models/querybuilder_sql.go index 58b6a6ab2ed..50fa7d7b5d3 100644 --- a/pkg/models/querybuilder_sql.go +++ b/pkg/models/querybuilder_sql.go @@ -332,16 +332,16 @@ func runCountQuery(query string, args []interface{}) (int, error) { return result.Int, nil } -func runSumQuery(query string, args []interface{}) (uint64, error) { +func runSumQuery(query string, args []interface{}) (float64, error) { // Perform query and fetch result result := struct { - Uint uint64 `db:"sum"` + Float64 float64 `db:"sum"` }{0} if err := database.DB.Get(&result, query, args...); err != nil && err != sql.ErrNoRows { return 0, err } - return result.Uint, nil + return result.Float64, nil } func executeFindQuery(tableName string, body string, args []interface{}, sortAndPagination string, whereClauses []string, havingClauses []string) ([]int, int) { From fdfc74d1e7fb81f4340d31c4bc9ccc56e0de53b3 Mon Sep 17 00:00:00 2001 From: Infinite Date: Tue, 15 Dec 2020 20:56:33 +0000 Subject: [PATCH 3/5] Add migration to reset scene sizes and scan logic to repopulate if empty --- pkg/database/migrations/16_reset_scene_size.up.sql | 1 + pkg/manager/task_scan.go | 6 +++--- ui/v2.5/src/components/Changelog/versions/v050.md | 3 +++ 3 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 pkg/database/migrations/16_reset_scene_size.up.sql diff --git a/pkg/database/migrations/16_reset_scene_size.up.sql b/pkg/database/migrations/16_reset_scene_size.up.sql new file mode 100644 index 00000000000..ba97c759303 --- /dev/null +++ b/pkg/database/migrations/16_reset_scene_size.up.sql @@ -0,0 +1 @@ +UPDATE `scenes` SET `size` = NULL; diff --git a/pkg/manager/task_scan.go b/pkg/manager/task_scan.go index aa7fe65f75b..b04e278b337 100644 --- a/pkg/manager/task_scan.go +++ b/pkg/manager/task_scan.go @@ -360,7 +360,7 @@ func (t *ScanTask) scanScene() *models.Scene { // if the mod time of the file is different than that of the associated // scene, then recalculate the checksum and regenerate the thumbnail modified := t.isFileModified(fileModTime, scene.FileModTime) - if modified { + if modified || !scene.Size.Valid { scene, err = t.rescanScene(scene, fileModTime) if err != nil { logger.Error(err.Error()) @@ -534,7 +534,7 @@ func (t *ScanTask) scanScene() *models.Scene { Height: sql.NullInt64{Int64: int64(videoFile.Height), Valid: true}, Framerate: sql.NullFloat64{Float64: videoFile.FrameRate, Valid: true}, Bitrate: sql.NullInt64{Int64: videoFile.Bitrate, Valid: true}, - Size: sql.NullString{String: strconv.Itoa(int(videoFile.Size)), Valid: true}, + Size: sql.NullString{String: strconv.FormatInt(videoFile.Size, 10), Valid: true}, FileModTime: models.NullSQLiteTimestamp{ Timestamp: fileModTime, Valid: true, @@ -610,7 +610,7 @@ func (t *ScanTask) rescanScene(scene *models.Scene, fileModTime time.Time) (*mod Height: &sql.NullInt64{Int64: int64(videoFile.Height), Valid: true}, Framerate: &sql.NullFloat64{Float64: videoFile.FrameRate, Valid: true}, Bitrate: &sql.NullInt64{Int64: videoFile.Bitrate, Valid: true}, - Size: &sql.NullString{String: strconv.Itoa(int(videoFile.Size)), Valid: true}, + Size: &sql.NullString{String: strconv.FormatInt(videoFile.Size, 10), Valid: true}, FileModTime: &models.NullSQLiteTimestamp{ Timestamp: fileModTime, Valid: true, diff --git a/ui/v2.5/src/components/Changelog/versions/v050.md b/ui/v2.5/src/components/Changelog/versions/v050.md index 548f9dbb836..bd5176b2580 100644 --- a/ui/v2.5/src/components/Changelog/versions/v050.md +++ b/ui/v2.5/src/components/Changelog/versions/v050.md @@ -1,3 +1,5 @@ +#### 💥 **Note: After upgrading, all scene file sizes will be 0B until a new [scan](/settings?tab=tasks) is run. + ### ✨ New Features * Add organized flag for scenes, galleries and images. * Allow configuration of visible navbar items. @@ -13,4 +15,5 @@ * Support configurable number of threads for scanning and generation. ### 🐛 Bug fixes +* Corrected file sizes on 32bit platforms * Fixed login redirect to remember the current page. From d17e9fc08eba5bb4b69ce947850b443a44f05d62 Mon Sep 17 00:00:00 2001 From: Infinite Date: Mon, 21 Dec 2020 01:40:19 +0000 Subject: [PATCH 4/5] Bump migration version --- pkg/database/database.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/database/database.go b/pkg/database/database.go index 389fe5bae8f..231fe22b7e8 100644 --- a/pkg/database/database.go +++ b/pkg/database/database.go @@ -20,7 +20,7 @@ import ( var DB *sqlx.DB var dbPath string -var appSchemaVersion uint = 16 +var appSchemaVersion uint = 17 var databaseSchemaVersion uint const sqlite3Driver = "sqlite3ex" From 0c7436c7bd3db1cbafaa2d7848415c85a49abce5 Mon Sep 17 00:00:00 2001 From: WithoutPants <53250216+WithoutPants@users.noreply.github.com> Date: Tue, 22 Dec 2020 09:40:27 +1100 Subject: [PATCH 5/5] Rename migration --- .../{16_reset_scene_size.up.sql => 17_reset_scene_size.up.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pkg/database/migrations/{16_reset_scene_size.up.sql => 17_reset_scene_size.up.sql} (100%) diff --git a/pkg/database/migrations/16_reset_scene_size.up.sql b/pkg/database/migrations/17_reset_scene_size.up.sql similarity index 100% rename from pkg/database/migrations/16_reset_scene_size.up.sql rename to pkg/database/migrations/17_reset_scene_size.up.sql