test: tests for offloading (#77)

Author: dkharms

.github/workflows/cd.yml

@@ -1,3 +1,5 @@
+# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json
+
 name: CD
 
 on:

.github/workflows/ci.yml

@@ -1,3 +1,5 @@
+# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json
+
 name: CI
 
 on:
@@ -14,6 +16,21 @@ jobs:
   test:
     runs-on: ubuntu-latest
     steps:
+      # GitHub Actions do not support passing arguments to containers (that are declared in the services section of a job),
+      # but MinIO depends on it (it needs `server` argument at least).
+      # When [this](https://github.com/actions/runner/pull/1152) will be merged we can get rid of this step.
+      - name: Setup MinIO
+        uses: addnab/docker-run-action@v3
+        with:
+          registry: quay.io
+          image: minio/minio:latest
+          options: |
+            --detach
+            --publish 9000:9000
+          run: |
+            minio \
+            server /data
+
       - name: Checkout code
         uses: actions/checkout@v5
 

Makefile

@@ -41,8 +41,12 @@ run: build-binaries
 push-image: build-image
 	docker push ${IMAGE}:${VERSION}
 
+.PHONY: test-deps
+test-deps:
+	@docker compose -f tests/docker-compose.yml up -d
+
 .PHONY: test
-test:
+test: test-deps
 	LOG_LEVEL=ERROR go test ./... -count 1
 
 .bin-deps: export GOBIN := $(LOCAL_BIN)
@@ -75,15 +79,15 @@ proto:
 
 # arg -count=1 is used to disable tests caching (it is necessary when we want reproduce bugs of broken test isolation)
 .PHONY: ci-tests
-ci-tests:
+ci-tests: test-deps
 	set -o pipefail ;\
 	go test -v -short -count=1 -coverpkg=github.com/ozontech/seq-db/... -covermode=atomic -coverprofile=cover-tmp.out ./... 2>&1 | \
 		tee /dev/stderr | go-junit-report -set-exit-code > junit.xml
 	grep -vE ".pb.go|pb.*.go" cover-tmp.out > cover.out || cp cover-tmp.out cover.out
 	go tool cover -func=./cover.out
 
 .PHONY: ci-tests-race
-ci-tests-race:
+ci-tests-race: test-deps
 	set -o pipefail ;\
 	go test -short ./... -count=1 -race
 

frac/remote.go

@@ -183,6 +183,10 @@ func (f *Remote) Suicide() {
 	}
 }
 
+func (f *Remote) String() string {
+	return fracToString(f, "remote")
+}
+
 func (f *Remote) createDataProvider(ctx context.Context) *sealedDataProvider {
 	return &sealedDataProvider{
 		ctx:              ctx,

fracmanager/config.go

@@ -28,6 +28,7 @@ type Config struct {
 	Fraction          frac.Config
 
 	OffloadingEnabled   bool
+	OffloadingForced    bool
 	OffloadingRetention time.Duration
 }
 

fracmanager/fracmanager.go

@@ -666,6 +666,19 @@ func (fm *FracManager) SealForcedForTests() {
 	}
 }
 
+func (fm *FracManager) OffloadForcedForTests() {
+	if !(fm.config.OffloadingEnabled && fm.config.OffloadingForced) {
+		panic("trying to force offloading when it is disabled")
+	}
+
+	// Offloading works only for sealed fractions.
+	fm.SealForcedForTests()
+
+	var wg sync.WaitGroup
+	fm.cleanupFractions(&wg)
+	wg.Wait()
+}
+
 func (fm *FracManager) ResetCacheForTests() {
 	fm.cacheMaintainer.Reset()
 }

fracmanager/proxy_frac.go

@@ -3,6 +3,7 @@ package fracmanager
 import (
 	"context"
 	"errors"
+	"fmt"
 	"sync"
 	"time"
 
@@ -195,6 +196,10 @@ func (f *proxyFrac) Suicide() {
 	}
 }
 
+func (f *proxyFrac) String() string {
+	return fmt.Sprintf("%s", f.cur())
+}
+
 func (f *proxyFrac) isActiveState() bool {
 	return f.active != nil && f.sealed == nil && !f.readonly
 }

fracmanager/sealed_frac_cache_test.go

@@ -365,9 +365,6 @@ func TestExtraFractionsRemoved(t *testing.T) {
 	assert.NoError(t, err)
 
 	dp := frac.NewDocProvider()
-	metaRoot := insaneJSON.Spawn()
-	defer insaneJSON.Release(metaRoot)
-
 	infos := map[string]*frac.Info{}
 
 	for i := 1; i < times+1; i++ {
@@ -398,6 +395,7 @@ func TestExtraFractionsRemoved(t *testing.T) {
 
 	fracsOnDisk := []string{}
 	fracCacheFromDisk, err := loadFracCache(dataDir)
+
 	assert.NoError(t, err)
 	for k := range fracCacheFromDisk {
 		fracsOnDisk = append(fracsOnDisk, k)

go.mod

@@ -5,7 +5,7 @@ go 1.24
 require (
 	contrib.go.opencensus.io/exporter/jaeger v0.2.1
 	github.com/KimMachineGun/automemlimit v0.7.4
-	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137
+	github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b
 	github.com/aws/aws-sdk-go-v2 v1.36.5
 	github.com/aws/aws-sdk-go-v2/config v1.29.17
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.70
@@ -60,23 +60,24 @@ require (
 	github.com/aws/smithy-go v1.22.4 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/felixge/fgprof v0.9.3 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/felixge/fgprof v0.9.5 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
+	github.com/google/pprof v0.0.0-20250422154841-e1f9c1950416 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
-	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.65.0 // indirect
 	github.com/prometheus/procfs v0.16.1 // indirect
+	github.com/rogpeppe/go-internal v1.13.1 // indirect
 	github.com/uber/jaeger-client-go v2.25.0+incompatible // indirect
-	golang.org/x/net v0.40.0 // indirect
+	golang.org/x/net v0.41.0 // indirect
 	golang.org/x/sys v0.33.0 // indirect
 	golang.org/x/text v0.26.0 // indirect
-	google.golang.org/api v0.95.0 // indirect
+	google.golang.org/api v0.230.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )