diff --git a/repository-inmemory/src/main/java/tech/ydb/yoj/repository/test/inmemory/InMemoryRepository.java b/repository-inmemory/src/main/java/tech/ydb/yoj/repository/test/inmemory/InMemoryRepository.java index 56bef103..cc3adfa5 100644 --- a/repository-inmemory/src/main/java/tech/ydb/yoj/repository/test/inmemory/InMemoryRepository.java +++ b/repository-inmemory/src/main/java/tech/ydb/yoj/repository/test/inmemory/InMemoryRepository.java @@ -21,11 +21,6 @@ public class InMemoryRepository implements Repository { @Getter(AccessLevel.PACKAGE) private volatile InMemoryStorage storage = new InMemoryStorage(); - @Override - public void dropDb() { - storage.dropDb(); - } - @Override public String makeSnapshot() { String snapshotId = UUID.randomUUID().toString(); @@ -38,25 +33,29 @@ public void loadSnapshot(String id) { storage = snapshots.get(id).createSnapshot(); } - @Override - public Set> tables() { - return Set.copyOf(storage.tables()); - } - @Override public RepositoryTransaction startTransaction(TxOptions options) { return new InMemoryRepositoryTransaction(options, this); } - public > SchemaOperations schema(TableDescriptor tableDescriptor) { - return new SchemaOperations() { + public SchemaOperations getSchemaOperations() { + return new SchemaOperations() { @Override - public void create() { + public void createTablespace() { + } + + @Override + public void removeTablespace() { + storage.dropDb(); + } + + @Override + public > void createTable(TableDescriptor tableDescriptor) { storage.createTable(tableDescriptor); } @Override - public void drop() { + public > void dropTable(TableDescriptor tableDescriptor) { if (!storage.dropTable(tableDescriptor)) { throw new DropTableException(String.format("Can't drop table %s: table doesn't exist", tableDescriptor.toDebugString()) @@ -65,9 +64,13 @@ public void drop() { } @Override - public boolean exists() { + public > boolean hasTable(TableDescriptor tableDescriptor) { return storage.containsTable(tableDescriptor); } }; } + + @Override + public void shutdown() { + } } diff --git a/repository-test/src/main/java/tech/ydb/yoj/repository/test/RepositoryTest.java b/repository-test/src/main/java/tech/ydb/yoj/repository/test/RepositoryTest.java index de7e6db2..023b5f03 100644 --- a/repository-test/src/main/java/tech/ydb/yoj/repository/test/RepositoryTest.java +++ b/repository-test/src/main/java/tech/ydb/yoj/repository/test/RepositoryTest.java @@ -23,6 +23,7 @@ import tech.ydb.yoj.repository.db.SchemaOperations; import tech.ydb.yoj.repository.db.StdTxManager; import tech.ydb.yoj.repository.db.Table; +import tech.ydb.yoj.repository.db.TableDescriptor; import tech.ydb.yoj.repository.db.Tx; import tech.ydb.yoj.repository.db.TxManager; import tech.ydb.yoj.repository.db.TxOptions; @@ -144,17 +145,21 @@ public void tearDown() { @Test public void schema() { - SchemaOperations schema = repository.schema(Simple.class); - schema.create(); - schema.create(); // second create doesn't fail - schema.drop(); - assertThatExceptionOfType(DropTableException.class).isThrownBy(schema::drop); // second drop fails + SchemaOperations schema = repository.getSchemaOperations(); + TableDescriptor tableDescriptor = TableDescriptor.from(EntitySchema.of(Simple.class)); + schema.createTable(tableDescriptor); + schema.createTable(tableDescriptor); // second create doesn't fail + schema.dropTable(tableDescriptor); + assertThatExceptionOfType(DropTableException.class) + .isThrownBy(() -> schema.dropTable(tableDescriptor)); // second drop fails } @Test public void multiLevelDirectorySchema() { - SchemaOperations schema = repository.schema(MultiLevelDirectory.class); - schema.create(); + TableDescriptor tableDescriptor = TableDescriptor.from( + EntitySchema.of(MultiLevelDirectory.class) + ); + repository.getSchemaOperations().createTable(tableDescriptor); } @Test @@ -163,7 +168,7 @@ public void snapshotWithSubfolders() { checkEmpty(txManager); - String initSnapshotId = repository.makeSnapshot(); + String initSnapshotId = repository.getSchemaOperations().makeSnapshot(); txManager.tx(() -> { TestEntityOperations db = BaseDb.current(TestEntityOperations.class); @@ -178,14 +183,14 @@ public void snapshotWithSubfolders() { checkNotEmpty(txManager); // make ne snapshot and load initial - String snapshotWithDataId = repository.makeSnapshot(); - repository.loadSnapshot(initSnapshotId); + String snapshotWithDataId = repository.getSchemaOperations().makeSnapshot(); + repository.getSchemaOperations().loadSnapshot(initSnapshotId); // must be empty after load of initial snapshot checkEmpty(txManager); // load snapshot created after inserts and check entities present - repository.loadSnapshot(snapshotWithDataId); + repository.getSchemaOperations().loadSnapshot(snapshotWithDataId); checkNotEmpty(txManager); } diff --git a/repository-test/src/main/java/tech/ydb/yoj/repository/test/RepositoryTestSupport.java b/repository-test/src/main/java/tech/ydb/yoj/repository/test/RepositoryTestSupport.java index 3b03e420..f9105489 100644 --- a/repository-test/src/main/java/tech/ydb/yoj/repository/test/RepositoryTestSupport.java +++ b/repository-test/src/main/java/tech/ydb/yoj/repository/test/RepositoryTestSupport.java @@ -25,15 +25,6 @@ public void setUp() { @After public void tearDown() { - clearDb(this.repository); - } - - public static void clearDb(Repository repo) { - Set> tableDescriptors = repo.tables(); - new StdTxManager(repo).tx(() -> { - for (TableDescriptor tableDescriptor : tableDescriptors) { - Tx.Current.get().getRepositoryTransaction().table(tableDescriptor).deleteAll(); - } - }); + this.repository.getSchemaOperations().removeTablespace(); } } diff --git a/repository-test/src/main/java/tech/ydb/yoj/repository/test/entity/TestEntities.java b/repository-test/src/main/java/tech/ydb/yoj/repository/test/entity/TestEntities.java index a78227bd..25d8dc9a 100644 --- a/repository-test/src/main/java/tech/ydb/yoj/repository/test/entity/TestEntities.java +++ b/repository-test/src/main/java/tech/ydb/yoj/repository/test/entity/TestEntities.java @@ -2,7 +2,9 @@ import lombok.NonNull; import tech.ydb.yoj.repository.db.Entity; +import tech.ydb.yoj.repository.db.EntitySchema; import tech.ydb.yoj.repository.db.Repository; +import tech.ydb.yoj.repository.db.SchemaOperations; import tech.ydb.yoj.repository.db.TableDescriptor; import tech.ydb.yoj.repository.test.sample.model.Book; import tech.ydb.yoj.repository.test.sample.model.Bubble; @@ -73,11 +75,14 @@ private TestEntities() { @SuppressWarnings("unchecked") public static Repository init(@NonNull Repository repository) { - repository.createTablespace(); - ALL.forEach(entityClass -> repository.schema(entityClass).create()); + SchemaOperations schemaOperations = repository.getSchemaOperations(); + + schemaOperations.createTablespace(); + + ALL.forEach(entityClass -> schemaOperations.createTable(TableDescriptor.from(EntitySchema.of(entityClass)))); for (TableDescriptor tableDescriptor : ALL_TABLE_DESCRIPTORS) { - repository.schema(tableDescriptor).create(); + schemaOperations.createTable(tableDescriptor); } return repository; diff --git a/repository-ydb-v2/src/main/java/tech/ydb/yoj/repository/ydb/YdbRepository.java b/repository-ydb-v2/src/main/java/tech/ydb/yoj/repository/ydb/YdbRepository.java index ff45b467..0accc3fb 100644 --- a/repository-ydb-v2/src/main/java/tech/ydb/yoj/repository/ydb/YdbRepository.java +++ b/repository-ydb-v2/src/main/java/tech/ydb/yoj/repository/ydb/YdbRepository.java @@ -57,8 +57,6 @@ public class YdbRepository implements Repository { private final GrpcTransport transport; private final CloseableMemoizer sessionClient; - private final ConcurrentMap> entityClassesByTableName; - public YdbRepository(@NonNull YdbConfig config) { this(config, NopAuthProvider.INSTANCE); } @@ -88,7 +86,6 @@ public YdbRepository(@NonNull YdbConfig config, @NonNull GrpcTransport transport } public YdbRepository(@NonNull YdbConfig config, @NonNull Settings repositorySettings, @NonNull GrpcTransport transport) { - this.entityClassesByTableName = new ConcurrentHashMap<>(); this.transport = transport; this.sessionClient = MoreSuppliers.memoizeCloseable(() -> new SessionClient(config, repositorySettings, transport)); } @@ -174,6 +171,7 @@ public SessionManager getSessionManager() { return sessionClient.get().sessionManager; } + @Override public YdbSchemaOperations getSchemaOperations() { return sessionClient.get().schemaOperations; } @@ -219,112 +217,11 @@ public void shutdown() { Exceptions.closeAll(sessionClient, transport); } - @Override - public void createTablespace() { - getSchemaOperations().createTablespace(); - } - - @Override - public Set> tables() { - return getSchemaOperations().getTableNames().stream() - .map(entityClassesByTableName::get) - .filter(Objects::nonNull) - .collect(toUnmodifiableSet()); - } - @Override public RepositoryTransaction startTransaction(TxOptions options) { return new YdbRepositoryTransaction<>(this, options); } - @Override - public String makeSnapshot() { - YdbSchemaOperations schemaOperations = getSchemaOperations(); - - String snapshotPath = schemaOperations.getTablespace() + ".snapshot-" + UUID.randomUUID() + "/"; - schemaOperations.snapshot(snapshotPath); - return snapshotPath; - } - - @Override - public void loadSnapshot(String id) { - YdbSchemaOperations schemaOperations = getSchemaOperations(); - - String current = schemaOperations.getTablespace(); - - schemaOperations.getTableNames().forEach(schemaOperations::dropTable); - schemaOperations.getDirectoryNames().stream() - .filter(name -> !schemaOperations.isSnapshotDirectory(name)) - .forEach(schemaOperations::removeDirectoryRecursive); - - schemaOperations.setTablespace(id); - schemaOperations.snapshot(current); - schemaOperations.setTablespace(current); - - // NB: We use getSessionManager() method to allow mocking YdbRepository - sessionClient.reset(); - } - - @Override - public void dropDb() { - try { - getSchemaOperations().removeTablespace(); - entityClassesByTableName.clear(); - } catch (Exception e) { - log.error("Could not drop all tables from tablespace", e); - } - } - - @Override - public > SchemaOperations schema(TableDescriptor tableDescriptor) { - EntitySchema schema = EntitySchema.of(tableDescriptor.entityType()); - return new SchemaOperations<>() { - @Override - public void create() { - String tableName = tableDescriptor.tableName(); - getSchemaOperations().createTable( - tableName, - schema.flattenFields(), - schema.flattenId(), - extractHint(), - schema.getGlobalIndexes(), - schema.getTtlModifier(), - schema.getChangefeeds() - ); - entityClassesByTableName.put(tableName, tableDescriptor); - } - - private YdbTableHint extractHint() { - try { - Field ydbTableHintField = tableDescriptor.entityType().getDeclaredField("ydbTableHint"); - ydbTableHintField.setAccessible(true); - return (YdbTableHint) ydbTableHintField.get(null); - } catch (NoSuchFieldException | IllegalAccessException ignored) { - return null; - } - } - - @Override - public void drop() { - String tableName = tableDescriptor.tableName(); - getSchemaOperations().dropTable(tableName); - entityClassesByTableName.remove(tableName); - } - - @Override - public boolean exists() { - String tableName = tableDescriptor.tableName(); - boolean exists = getSchemaOperations().hasTable(tableName); - if (exists) { - entityClassesByTableName.put(tableName, tableDescriptor); - } else { - entityClassesByTableName.remove(tableName); - } - return exists; - } - }; - } - @Value public static class Query { Statement statement; diff --git a/repository-ydb-v2/src/main/java/tech/ydb/yoj/repository/ydb/client/YdbSchemaOperations.java b/repository-ydb-v2/src/main/java/tech/ydb/yoj/repository/ydb/client/YdbSchemaOperations.java index 5a25910e..7f170874 100644 --- a/repository-ydb-v2/src/main/java/tech/ydb/yoj/repository/ydb/client/YdbSchemaOperations.java +++ b/repository-ydb-v2/src/main/java/tech/ydb/yoj/repository/ydb/client/YdbSchemaOperations.java @@ -33,7 +33,11 @@ import tech.ydb.yoj.InternalApi; import tech.ydb.yoj.databind.schema.Changefeed.Consumer.Codec; import tech.ydb.yoj.databind.schema.Schema; +import tech.ydb.yoj.repository.db.Entity; import tech.ydb.yoj.repository.db.EntitySchema; +import tech.ydb.yoj.repository.db.SchemaOperations; +import tech.ydb.yoj.repository.db.Snapshotter; +import tech.ydb.yoj.repository.db.TableDescriptor; import tech.ydb.yoj.repository.db.exception.CreateTableException; import tech.ydb.yoj.repository.db.exception.DropTableException; import tech.ydb.yoj.repository.ydb.exception.SnapshotCreateException; @@ -42,10 +46,12 @@ import tech.ydb.yoj.repository.ydb.yql.YqlPrimitiveType; import tech.ydb.yoj.repository.ydb.yql.YqlType; +import java.lang.reflect.Field; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.UUID; import java.util.function.Function; import java.util.stream.Stream; @@ -57,7 +63,7 @@ import static tech.ydb.core.StatusCode.SCHEME_ERROR; @InternalApi -public final class YdbSchemaOperations { +public final class YdbSchemaOperations implements SchemaOperations { private static final Logger log = LoggerFactory.getLogger(YdbSchemaOperations.class); @Getter @@ -78,6 +84,42 @@ public YdbSchemaOperations( this.topicClient = topicClient; } + @Override + public > void createTable(TableDescriptor tableDescriptor) { + String tableName = tableDescriptor.tableName(); + var schema = EntitySchema.of(tableDescriptor.entityType()); + createTable( + tableName, + schema.flattenFields(), + schema.flattenId(), + extractHint(tableDescriptor), + schema.getGlobalIndexes(), + schema.getTtlModifier(), + schema.getChangefeeds() + ); + } + + private static > YdbTableHint extractHint(TableDescriptor tableDescriptor) { + try { + Field ydbTableHintField = tableDescriptor.entityType().getDeclaredField("ydbTableHint"); + ydbTableHintField.setAccessible(true); + return (YdbTableHint) ydbTableHintField.get(null); + } catch (NoSuchFieldException | IllegalAccessException ignored) { + return null; + } + } + + @Override + public > void dropTable(TableDescriptor tableDescriptor) { + String tableName = tableDescriptor.tableName(); + dropTable(tableName); + } + + @Override + public > boolean hasTable(TableDescriptor tableDescriptor) { + return hasPath(tablespace + tableDescriptor.tableName()); + } + public void setTablespace(String tablespace) { this.tablespace = YdbPaths.canonicalTablespace(tablespace); } @@ -248,10 +290,6 @@ public Table describeTable( return new Table(tablespace + name, ydbColumns, ydbIndexes, tableTtl); } - public boolean hasTable(String name) { - return hasPath(tablespace + name); - } - public void dropTable(String name) { dropTablePath(tablespace + name); } @@ -479,6 +517,35 @@ public boolean hasPath(String path) { throw new YdbRepositoryException("Can't describe table '" + path + "': " + result); } + @Override + public String makeSnapshot() { + String snapshotPath = tablespace + ".snapshot-" + UUID.randomUUID() + "/"; + snapshot(snapshotPath); + return snapshotPath; + } + + @Override + public void loadSnapshot(String id) { + String current = tablespace; + + for (String s : getTableNames()) { + dropTable(s); + } + + for (String name : getDirectoryNames()) { + if (!isSnapshotDirectory(name)) { + removeDirectoryRecursive(name); + } + } + + setTablespace(id); + snapshot(current); + setTablespace(current); + + // NB: We use getSessionManager() method to allow mocking YdbRepository + sessionClient.reset(); + } + @Value private static class DirectoryEntity { EntryType type; diff --git a/repository-ydb-v2/src/test/java/tech/ydb/yoj/repository/ydb/YdbRepositoryIntegrationTest.java b/repository-ydb-v2/src/test/java/tech/ydb/yoj/repository/ydb/YdbRepositoryIntegrationTest.java index da77ea7b..8e1e26dd 100644 --- a/repository-ydb-v2/src/test/java/tech/ydb/yoj/repository/ydb/YdbRepositoryIntegrationTest.java +++ b/repository-ydb-v2/src/test/java/tech/ydb/yoj/repository/ydb/YdbRepositoryIntegrationTest.java @@ -55,6 +55,7 @@ import tech.ydb.yoj.repository.db.RecordEntity; import tech.ydb.yoj.repository.db.Repository; import tech.ydb.yoj.repository.db.RepositoryTransaction; +import tech.ydb.yoj.repository.db.SchemaOperations; import tech.ydb.yoj.repository.db.StdTxManager; import tech.ydb.yoj.repository.db.TableDescriptor; import tech.ydb.yoj.repository.db.Tx; @@ -137,12 +138,12 @@ public class YdbRepositoryIntegrationTest extends RepositoryTest { @Override protected Repository createRepository() { - Repository repository = super.createRepository(); - repository.schema(NonSerializableEntity.class).create(); - repository.schema(WithUnflattenableField.class).create(); - repository.schema(SubdirEntity.class).create(); - repository.schema(TtlEntity.class).create(); - repository.schema(ChangefeedEntity.class).create(); + SchemaOperations schema = super.createRepository().getSchemaOperations(); + schema.createTable(TableDescriptor.from(EntitySchema.of(NonSerializableEntity.class))); + schema.createTable(TableDescriptor.from(EntitySchema.of(WithUnflattenableField.class))); + schema.createTable(TableDescriptor.from(EntitySchema.of(SubdirEntity.class))); + schema.createTable(TableDescriptor.from(EntitySchema.of(TtlEntity.class))); + schema.createTable(TableDescriptor.from(EntitySchema.of(ChangefeedEntity.class))); return repository; } @@ -1004,10 +1005,11 @@ private void runWithModifiedStatusCode(StatusCodesProtos.StatusIds.StatusCode co @Test public void schemaWithHint() { - repository.schema(HintInt64Range.class).create(); - repository.schema(HintUniform.class).create(); - repository.schema(HintTablePreset.class).create(); - repository.schema(HintAutoPartitioningByLoad.class).create(); + SchemaOperations schema = repository.getSchemaOperations(); + schema.createTable(TableDescriptor.from(EntitySchema.of(HintInt64Range.class))); + schema.createTable(TableDescriptor.from(EntitySchema.of(HintUniform.class))); + schema.createTable(TableDescriptor.from(EntitySchema.of(HintTablePreset.class))); + schema.createTable(TableDescriptor.from(EntitySchema.of(HintAutoPartitioningByLoad.class))); } @Test @@ -1354,12 +1356,20 @@ public void transactionalTopicWritesRollbackReadNothing() { @Test public void schemaExistsForExistingEntity() { - assertThat(repository.schema(NonSerializableEntity.class).exists()).isTrue(); + SchemaOperations schema = repository.getSchemaOperations(); + TableDescriptor descriptor = TableDescriptor.from( + EntitySchema.of(NonSerializableEntity.class) + ); + assertThat(schema.hasTable(descriptor)).isTrue(); } @Test public void schemaNotExistsForMissingEntity() { - assertThat(repository.schema(MissingEntity.class).exists()).isFalse(); + SchemaOperations schema = repository.getSchemaOperations(); + TableDescriptor descriptor = TableDescriptor.from( + EntitySchema.of(MissingEntity.class) + ); + assertThat(schema.hasTable(descriptor)).isFalse(); } private List readAll(TopicClient topicClient, String topicPath, String consumer, String reader) { diff --git a/repository-ydb-v2/src/test/java/tech/ydb/yoj/repository/ydb/statement/AbstractMultipleVarsYqlStatementIntegrationTestBase.java b/repository-ydb-v2/src/test/java/tech/ydb/yoj/repository/ydb/statement/AbstractMultipleVarsYqlStatementIntegrationTestBase.java index 009f6f31..ce6f21c0 100644 --- a/repository-ydb-v2/src/test/java/tech/ydb/yoj/repository/ydb/statement/AbstractMultipleVarsYqlStatementIntegrationTestBase.java +++ b/repository-ydb-v2/src/test/java/tech/ydb/yoj/repository/ydb/statement/AbstractMultipleVarsYqlStatementIntegrationTestBase.java @@ -11,9 +11,11 @@ import tech.ydb.yoj.databind.schema.Table; import tech.ydb.yoj.repository.BaseDb; import tech.ydb.yoj.repository.db.Entity; +import tech.ydb.yoj.repository.db.EntitySchema; import tech.ydb.yoj.repository.db.Repository; import tech.ydb.yoj.repository.db.RepositoryTransaction; import tech.ydb.yoj.repository.db.StdTxManager; +import tech.ydb.yoj.repository.db.TableDescriptor; import tech.ydb.yoj.repository.db.TxManager; import tech.ydb.yoj.repository.db.TxOptions; import tech.ydb.yoj.repository.test.RepositoryTestSupport; @@ -53,8 +55,8 @@ static class RepositoryTransactionImpl extends YdbRepositoryTransaction> SchemaOperations schema(Class c) { - return schema(TableDescriptor.from(EntitySchema.of(c))); - } - - > SchemaOperations schema(TableDescriptor c); - - /** - * @deprecated For testing purposes only. Will only reliably work for tables that were created or inspected - * using calls to {@link #schema(Class)}. - */ - @Deprecated - Set> tables(); - default RepositoryTransaction startTransaction() { return startTransaction(IsolationLevel.SERIALIZABLE_READ_WRITE); } @@ -33,18 +17,13 @@ default RepositoryTransaction startTransaction(IsolationLevel isolationLevel) { return startTransaction(TxOptions.create(isolationLevel)); } - RepositoryTransaction startTransaction(TxOptions options); - - void dropDb(); - - String makeSnapshot(); + SchemaOperations getSchemaOperations(); - void loadSnapshot(String id); + RepositoryTransaction startTransaction(TxOptions options); default boolean healthCheck() { return true; } - default void shutdown() { - } + void shutdown(); } diff --git a/repository/src/main/java/tech/ydb/yoj/repository/db/SchemaOperations.java b/repository/src/main/java/tech/ydb/yoj/repository/db/SchemaOperations.java index 39cc351d..86cd91e6 100644 --- a/repository/src/main/java/tech/ydb/yoj/repository/db/SchemaOperations.java +++ b/repository/src/main/java/tech/ydb/yoj/repository/db/SchemaOperations.java @@ -1,12 +1,20 @@ package tech.ydb.yoj.repository.db; -public interface SchemaOperations { - void create(); +public interface SchemaOperations { + String makeSnapshot(); + + void loadSnapshot(String id); + + void createTablespace(); + + void removeTablespace(); + + > void createTable(TableDescriptor tableDescriptor); /** - * Drops the table. Does nothing if the table does not {@link #exists() exist}. + * Drops the table. Does nothing if the table does not {@link #hasTable(TableDescriptor tableDescriptor) exist}. */ - void drop(); + > void dropTable(TableDescriptor tableDescriptor); - boolean exists(); + > boolean hasTable(TableDescriptor tableDescriptor); }