From 3c93fb4d8f09de1948c1b65ea313531c567868c8 Mon Sep 17 00:00:00 2001 From: imbajin Date: Thu, 12 Jun 2025 14:08:24 +0800 Subject: [PATCH 01/51] Create auto-pr-review.yml --- .github/workflows/auto-pr-review.yml | 35 ++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/auto-pr-review.yml diff --git a/.github/workflows/auto-pr-review.yml b/.github/workflows/auto-pr-review.yml new file mode 100644 index 0000000000..6a585355f7 --- /dev/null +++ b/.github/workflows/auto-pr-review.yml @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: "Auto PR Commenter" + +on: + pull_request_target: + types: [opened] + +jobs: + add-review-comment: + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - name: Add review comment + uses: peter-evans/create-or-update-comment@v4 + with: + issue-number: ${{ github.event.pull_request.number }} + body: | + @codecov-ai-reviewer review From b385b9ac4d0212dfd12996c548499969eef766a7 Mon Sep 17 00:00:00 2001 From: gydeng Date: Fri, 13 Jun 2025 06:43:23 +0000 Subject: [PATCH 02/51] feat(backend): Add RocksDB Plus (ToplingDB) support --- .../conf/graphs/db_bench_community.yaml | 135 ++++++++++++++++++ .../static/conf/graphs/hugegraph.properties | 1 + hugegraph-server/hugegraph-rocksdb/pom.xml | 2 +- .../backend/store/rocksdb/OpenedRocksDB.java | 8 +- .../backend/store/rocksdb/RocksDBOptions.java | 8 ++ .../store/rocksdb/RocksDBStdSessions.java | 96 +++++++++++-- .../backend/store/rocksdb/RocksDBStore.java | 19 +-- .../store/rocksdbsst/RocksDBSstStore.java | 2 +- .../unit/rocksdb/BaseRocksDBUnitTest.java | 2 +- .../unit/rocksdb/RocksDBSessionsTest.java | 3 +- 10 files changed, 250 insertions(+), 26 deletions(-) create mode 100644 hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/db_bench_community.yaml diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/db_bench_community.yaml b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/db_bench_community.yaml new file mode 100644 index 0000000000..86e74a19cf --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/db_bench_community.yaml @@ -0,0 +1,135 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# common parameters +http: + # normally parent path of db path + document_root: /dev/shm/db_bench_community + listening_ports: '2011' +setenv: + StrSimpleEnvNameNotOverwrite: StringValue + IntSimpleEnvNameNotOverwrite: 16384 + OverwriteThisEnv: + #comment: overwrite is default to false + overwrite: true + value: force overwrite this env by overwrite true +Cache: + lru_cache: + class: LRUCache + params: + capacity: 8G + num_shard_bits: -1 + strict_capacity_limit: false + high_pri_pool_ratio: 0.5 + use_adaptive_mutex: false + metadata_charge_policy: kFullChargeCacheMetadata +Statistics: + stat: + class: default + params: + discard_tickers: + - rocksdb.block.cache + - rocksdb.block.cachecompressed + - rocksdb.block + - rocksdb.memtable.payload.bytes.at.flush + - rocksdb.memtable.garbage.bytes.at.flush + - rocksdb.txn + - rocksdb.blobdb + - rocksdb.row.cache + - rocksdb.number.block + - rocksdb.bloom.filter + - rocksdb.persistent + - rocksdb.sim.block.cache + discard_histograms: + # comment: .... + - rocksdb.blobdb + - rocksdb.bytes.compressed + - rocksdb.bytes.decompressed + - rocksdb.num.index.and.filter.blocks.read.per.level + - rocksdb.num.data.blocks.read.per.level + - rocksdb.compression.times.nanos + - rocksdb.decompression.times.nanos + - rocksdb.read.block.get.micros + - rocksdb.write.raw.block.micros + # comment end of array + #stats_level: kAll + stats_level: kDisableAll +MemTableRepFactory: + cspp: + class: cspp + params: + mem_cap: 2G + use_vm: false + token_use_idle: true + skiplist: + class: SkipList + params: + lookahead: 0 +TableFactory: + bb: + class: BlockBasedTable + params: + checksum: kCRC32c + block_size: 4K + block_restart_interval: 16 + index_block_restart_interval: 1 + metadata_block_size: 4K + enable_index_compression: true + block_cache: "${lru_cache}" + block_cache_compressed: + persistent_cache: + filter_policy: + dispatch: + class: DispatcherTable + params: + default: bb + readers: + BlockBasedTable: bb + level_writers: [ bb, bb, bb, bb, bb, bb ] +CFOptions: + default: + max_write_buffer_number: 4 + memtable_factory: "${cspp}" + write_buffer_size: 8M + # set target_file_size_base as small as 512K is to make many SST files, + # thus key prefix cache can present efficiency + target_file_size_base: 512K + target_file_size_multiplier: 1 + table_factory: dispatch + max_bytes_for_level_base: 4M + max_bytes_for_level_multiplier: 4 + level_compaction_dynamic_level_bytes: false + level0_slowdown_writes_trigger: 20 + level0_stop_writes_trigger: 36 + level0_file_num_compaction_trigger: 4 + compression_per_level: + - kNoCompression + - kNoCompression + - kZSTD + - kZSTD + - kZSTD + - kZSTD + - kZSTD +DBOptions: + dbo: + create_if_missing: true + create_missing_column_families: false # this is important, must be false to hugegraph + max_background_compactions: 13 + max_subcompactions: 1 + max_level1_subcompactions: 7 + inplace_update_support: false + WAL_size_limit_MB: 0 + statistics: "${stat}" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/hugegraph.properties b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/hugegraph.properties index b77cacb2de..23617106b7 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/hugegraph.properties +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/hugegraph.properties @@ -42,6 +42,7 @@ search.text_analyzer_mode=INDEX # rocksdb backend config #rocksdb.data_path=/path/to/disk #rocksdb.wal_path=/path/to/disk +#rocksdb.option_path=./db_bench_community.yaml # hbase backend config #hbase.hosts=localhost diff --git a/hugegraph-server/hugegraph-rocksdb/pom.xml b/hugegraph-server/hugegraph-rocksdb/pom.xml index 3cf448c9cb..c749254962 100644 --- a/hugegraph-server/hugegraph-rocksdb/pom.xml +++ b/hugegraph-server/hugegraph-rocksdb/pom.xml @@ -37,7 +37,7 @@ org.rocksdb rocksdbjni - 7.2.2 + 8.10.2-SNAPSHOT diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/OpenedRocksDB.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/OpenedRocksDB.java index 2aa6d2292c..8c814206f4 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/OpenedRocksDB.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/OpenedRocksDB.java @@ -34,6 +34,7 @@ import org.rocksdb.Checkpoint; import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.RocksDB; +import org.rocksdb.SidePluginRepo; import org.rocksdb.SstFileManager; import org.slf4j.Logger; @@ -44,12 +45,14 @@ public class OpenedRocksDB implements AutoCloseable { private final RocksDB rocksdb; private final Map cfHandles; private final SstFileManager sstFileManager; + private final SidePluginRepo repo; public OpenedRocksDB(RocksDB rocksdb, Map cfHandles, - SstFileManager sstFileManager) { + SstFileManager sstFileManager, SidePluginRepo repo) { this.rocksdb = rocksdb; this.cfHandles = cfHandles; this.sstFileManager = sstFileManager; + this.repo = repo; } protected final RocksDB rocksdb() { @@ -90,6 +93,9 @@ public void close() { } this.cfHandles.clear(); + if (repo != null) { + this.repo.closeAllDB(); + } this.rocksdb.close(); } diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java index 17148e0944..67052373a5 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java @@ -86,6 +86,14 @@ public static synchronized RocksDBOptions instance() { "rocksdb-data/wal" ); + public static final ConfigOption OPTION_PATH = + new ConfigOption<>( + "rocksdb.option_path", + "The path of the JSON file for configuring RocksDB options.", + null, + "" + ); + public static final ConfigOption SST_PATH = new ConfigOption<>( "rocksdb.sst_path", diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java index c1cc1c5075..fe4318ec9a 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java @@ -31,6 +31,7 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.hugegraph.backend.BackendException; import org.apache.hugegraph.backend.serializer.BinarySerializer; @@ -63,6 +64,7 @@ import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; import org.rocksdb.RocksIterator; +import org.rocksdb.SidePluginRepo; import org.rocksdb.SstFileManager; import org.rocksdb.TableFormatConfig; import org.rocksdb.WriteBatch; @@ -71,6 +73,8 @@ import com.google.common.collect.ImmutableList; +import net.minidev.json.JSONObject; + public class RocksDBStdSessions extends RocksDBSessions { private static final Logger LOG = Log.logger(RocksDBStdSessions.class); @@ -78,28 +82,33 @@ public class RocksDBStdSessions extends RocksDBSessions { private final HugeConfig config; private final String dataPath; private final String walPath; + private final String optionPath; private volatile OpenedRocksDB rocksdb; private final AtomicInteger refCount; public RocksDBStdSessions(HugeConfig config, String database, String store, - String dataPath, String walPath) throws RocksDBException { + String dataPath, String walPath, String optionPath) throws + RocksDBException { super(config, database, store); this.config = config; this.dataPath = dataPath; this.walPath = walPath; - this.rocksdb = RocksDBStdSessions.openRocksDB(config, dataPath, walPath); + this.optionPath = optionPath; + this.rocksdb = RocksDBStdSessions.openRocksDB(config, dataPath, walPath, optionPath); this.refCount = new AtomicInteger(1); } public RocksDBStdSessions(HugeConfig config, String database, String store, String dataPath, String walPath, - List cfNames) throws RocksDBException { + List cfNames, String optionPath) throws RocksDBException { super(config, database, store); this.config = config; this.dataPath = dataPath; this.walPath = walPath; - this.rocksdb = RocksDBStdSessions.openRocksDB(config, cfNames, dataPath, walPath); + this.optionPath = optionPath; + this.rocksdb = + RocksDBStdSessions.openRocksDB(config, cfNames, dataPath, walPath, optionPath); this.refCount = new AtomicInteger(1); this.ingestExternalFile(); @@ -111,6 +120,7 @@ private RocksDBStdSessions(HugeConfig config, String database, String store, this.config = config; this.dataPath = origin.dataPath; this.walPath = origin.walPath; + this.optionPath = origin.optionPath; this.rocksdb = origin.rocksdb; this.refCount = origin.refCount; this.refCount.incrementAndGet(); @@ -207,7 +217,7 @@ public void reloadRocksDB() throws RocksDBException { this.rocksdb.close(); } this.rocksdb = RocksDBStdSessions.openRocksDB(this.config, ImmutableList.of(), - this.dataPath, this.walPath); + this.dataPath, this.walPath, this.optionPath); } @Override @@ -301,7 +311,7 @@ public String buildSnapshotPath(String snapshotPrefix) { public String hardLinkSnapshot(String snapshotPath) throws RocksDBException { String snapshotLinkPath = this.dataPath + "_temp"; try (OpenedRocksDB rocksdb = openRocksDB(this.config, ImmutableList.of(), - snapshotPath, null)) { + snapshotPath, null, this.optionPath)) { rocksdb.createCheckpoint(snapshotLinkPath); } LOG.info("The snapshot {} has been hard linked to {}", snapshotPath, snapshotLinkPath); @@ -366,7 +376,8 @@ private void ingestExternalFile() throws RocksDBException { } private static OpenedRocksDB openRocksDB(HugeConfig config, String dataPath, - String walPath) throws RocksDBException { + String walPath, String optionPath) throws + RocksDBException { // Init options Options options = new Options(); RocksDBStdSessions.initOptions(config, options, options, options, options); @@ -377,14 +388,29 @@ private static OpenedRocksDB openRocksDB(HugeConfig config, String dataPath, * Open RocksDB at the first time * Don't merge old CFs, we expect a clear DB when using this one */ - RocksDB rocksdb = RocksDB.open(options, dataPath); + RocksDB rocksdb = null; + SidePluginRepo repo = null; + // use rocksdb + if (StringUtils.isEmpty(optionPath)) { + LOG.info("Use rocksdb open default CF"); + rocksdb = RocksDB.open(options, dataPath); + } else { // use topling + LOG.info("Use topling open default CF"); + repo = new SidePluginRepo(); + String dbName = getDbName(dataPath); + repo.put(dbName, options); + repo.importAutoFile(optionPath); + rocksdb = repo.openDB(converseOptionsToJsonString(dataPath, null)); + } + Map cfs = new ConcurrentHashMap<>(); - return new OpenedRocksDB(rocksdb, cfs, sstFileManager); + return new OpenedRocksDB(rocksdb, cfs, sstFileManager, repo); } private static OpenedRocksDB openRocksDB(HugeConfig config, List cfNames, String dataPath, - String walPath) throws RocksDBException { + String walPath, String optionPath) throws + RocksDBException { // Old CFs should always be opened Set mergedCFs = RocksDBStdSessions.mergeOldCFs(dataPath, cfNames); @@ -407,10 +433,23 @@ private static OpenedRocksDB openRocksDB(HugeConfig config, } SstFileManager sstFileManager = new SstFileManager(Env.getDefault()); options.setSstFileManager(sstFileManager); - // Open RocksDB with CFs List cfhs = new ArrayList<>(); - RocksDB rocksdb = RocksDB.open(options, dataPath, cfds, cfhs); + + RocksDB rocksdb = null; + SidePluginRepo repo = null; + // use rocksdb + if (StringUtils.isEmpty(optionPath)) { + LOG.info("Use rocksdb open multi CFs"); + rocksdb = RocksDB.open(options, dataPath, cfds, cfhs); + } else { // use topling + LOG.info("Use topling open multi CFs"); + repo = new SidePluginRepo(); + String dbName = getDbName(dataPath); + repo.put(dbName, options); + repo.importAutoFile(optionPath); + rocksdb = repo.openDB(converseOptionsToJsonString(dataPath, cfs), cfhs); + } E.checkState(cfhs.size() == cfs.size(), "Expect same size of cf-handles and cf-names"); @@ -419,7 +458,7 @@ private static OpenedRocksDB openRocksDB(HugeConfig config, for (int i = 0; i < cfs.size(); i++) { cfHandles.put(cfs.get(i), new OpenedRocksDB.CFHandle(rocksdb, cfhs.get(i))); } - return new OpenedRocksDB(rocksdb, cfHandles, sstFileManager); + return new OpenedRocksDB(rocksdb, cfHandles, sstFileManager, repo); } private static Set mergeOldCFs(String path, @@ -443,6 +482,37 @@ public static Set listCFs(String path) throws RocksDBException { return cfs; } + private static String converseOptionsToJsonString(String dataPath, List cfs) { + // construct CFOptions + JSONObject columnFamilies = new JSONObject(); + // multi CFs + if (cfs != null) { + for (String cf : cfs) { + columnFamilies.put(cf, "$default"); + } + } else { // single default CF + columnFamilies.put("default", "$default"); + } + + // construct params + JSONObject params = new JSONObject(); + params.put("db_options", "$dbo"); + params.put("cf_options", "$default"); + params.put("column_families", columnFamilies); + params.put("path", dataPath); + + // construct wrapper + JSONObject wrapper = new JSONObject(); + wrapper.put("method", "DB::Open"); + wrapper.put("params", params); + + return wrapper.toString(); + } + + private static String getDbName(String dataPath) { + return dataPath.substring(dataPath.lastIndexOf("/") + 1); + } + public static void initOptions(HugeConfig conf, DBOptionsInterface db, MutableDBOptionsInterface mdb, diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStore.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStore.java index c34d9632fc..e4a5aaa002 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStore.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStore.java @@ -240,7 +240,7 @@ public synchronized void open(HugeConfig config) { openedDisks.add(disk); List tables = this.tableNames(e.getKey()); futures.add(openPool.submit(() -> { - this.open(config, disk, disk, tables); + this.open(config, disk, disk, tables, null); })); } } @@ -301,16 +301,17 @@ private void shutdownOpenPool(ExecutorService openPool) { protected RocksDBSessions open(HugeConfig config, List tableNames) { String dataPath = this.wrapPath(config.get(RocksDBOptions.DATA_PATH)); String walPath = this.wrapPath(config.get(RocksDBOptions.WAL_PATH)); - return this.open(config, dataPath, walPath, tableNames); + String optionPath = config.get(RocksDBOptions.OPTION_PATH); + return this.open(config, dataPath, walPath, tableNames, optionPath); } protected RocksDBSessions open(HugeConfig config, String dataPath, - String walPath, List tableNames) { + String walPath, List tableNames, String optionPath) { LOG.info("Opening RocksDB with data path: {}", dataPath); RocksDBSessions sessions = null; try { sessions = this.openSessionPool(config, dataPath, - walPath, tableNames); + walPath, tableNames, optionPath); } catch (RocksDBException e) { RocksDBSessions origin = this.dbs.get(dataPath); if (origin != null) { @@ -343,7 +344,7 @@ protected RocksDBSessions open(HugeConfig config, String dataPath, none = null; } try { - sessions = this.openSessionPool(config, dataPath, walPath, none); + sessions = this.openSessionPool(config, dataPath, walPath, none, optionPath); } catch (RocksDBException e1) { e = e1; } @@ -372,12 +373,14 @@ protected RocksDBSessions open(HugeConfig config, String dataPath, protected RocksDBSessions openSessionPool(HugeConfig config, String dataPath, String walPath, - List tableNames) throws RocksDBException { + List tableNames, String optionPath) throws + RocksDBException { if (tableNames == null) { - return new RocksDBStdSessions(config, this.database, this.store, dataPath, walPath); + return new RocksDBStdSessions(config, this.database, this.store, dataPath, walPath, + optionPath); } else { return new RocksDBStdSessions(config, this.database, this.store, - dataPath, walPath, tableNames); + dataPath, walPath, tableNames, optionPath); } } diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdbsst/RocksDBSstStore.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdbsst/RocksDBSstStore.java index d7b75610f1..e824cb0ee6 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdbsst/RocksDBSstStore.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdbsst/RocksDBSstStore.java @@ -38,7 +38,7 @@ public RocksDBSstStore(final BackendStoreProvider provider, @Override protected RocksDBSessions openSessionPool(HugeConfig config, String dataPath, String walPath, - List tableNames) + List tableNames, String optionPath) throws RocksDBException { if (tableNames == null) { return new RocksDBSstSessions(config, this.database(), this.store(), dataPath); diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/BaseRocksDBUnitTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/BaseRocksDBUnitTest.java index eb25fd8857..4658c686cd 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/BaseRocksDBUnitTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/BaseRocksDBUnitTest.java @@ -109,7 +109,7 @@ protected static long getLong(byte[] bytes) { private static RocksDBSessions open(String table) throws RocksDBException { HugeConfig config = FakeObjects.newConfig(); RocksDBSessions rocks = new RocksDBStdSessions(config, "db", "store", - DB_PATH, DB_PATH); + DB_PATH, DB_PATH, null); rocks.createTable(table); return rocks; } diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/RocksDBSessionsTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/RocksDBSessionsTest.java index 5dbc96c5c6..6327eb06d7 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/RocksDBSessionsTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/RocksDBSessionsTest.java @@ -190,7 +190,8 @@ public void testIngestSst() throws RocksDBException { Assert.assertFalse(sstSessions.existsTable(TABLE1)); Assert.assertFalse(sstSessions.existsTable(TABLE2)); - RocksDBSessions rocks = new RocksDBStdSessions(config, "db", "store", sstPath, sstPath); + RocksDBSessions rocks = + new RocksDBStdSessions(config, "db", "store", sstPath, sstPath, null); // Will ingest sst file of TABLE1 rocks.createTable(TABLE1); Assert.assertEquals(ImmutableList.of("1000"), From c26d84d7edf2d9543f109a22f66f1bc07f2158cc Mon Sep 17 00:00:00 2001 From: gydeng Date: Tue, 17 Jun 2025 09:19:52 +0000 Subject: [PATCH 03/51] fix(server): Use reflection to increase compatibility with old rocksdb packages --- .../backend/store/rocksdb/OpenedRocksDB.java | 20 ++- .../store/rocksdb/RocksDBStdSessions.java | 132 +++++++++++++++--- 2 files changed, 125 insertions(+), 27 deletions(-) diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/OpenedRocksDB.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/OpenedRocksDB.java index 8c814206f4..e83ba04dc4 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/OpenedRocksDB.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/OpenedRocksDB.java @@ -19,6 +19,7 @@ import java.io.File; import java.io.IOException; +import java.lang.reflect.Method; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Map; @@ -34,7 +35,6 @@ import org.rocksdb.Checkpoint; import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.RocksDB; -import org.rocksdb.SidePluginRepo; import org.rocksdb.SstFileManager; import org.slf4j.Logger; @@ -45,10 +45,10 @@ public class OpenedRocksDB implements AutoCloseable { private final RocksDB rocksdb; private final Map cfHandles; private final SstFileManager sstFileManager; - private final SidePluginRepo repo; + private final Object repo; public OpenedRocksDB(RocksDB rocksdb, Map cfHandles, - SstFileManager sstFileManager, SidePluginRepo repo) { + SstFileManager sstFileManager, Object repo) { this.rocksdb = rocksdb; this.cfHandles = cfHandles; this.sstFileManager = sstFileManager; @@ -94,7 +94,19 @@ public void close() { this.cfHandles.clear(); if (repo != null) { - this.repo.closeAllDB(); + LOG.info("SidePluginRepo instance found, attempting to call closeAllDB()."); + try { + // Get the class of the repo object at runtime. + Class sidePluginRepoClass = repo.getClass(); + Method closeAllDBMethod = sidePluginRepoClass.getMethod("closeAllDB"); + + // Invoke the method on the repo instance. + closeAllDBMethod.invoke(repo); + } catch (Exception e) { + // Catch potential reflection exceptions (e.g., NoSuchMethodException) + // and log them. This is safer than letting them crash the application. + LOG.error("Failed to reflectively call closeAllDB() on SidePluginRepo.", e); + } } this.rocksdb.close(); } diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java index fe4318ec9a..ccaed5de19 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java @@ -18,6 +18,9 @@ package org.apache.hugegraph.backend.store.rocksdb; import java.io.File; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -64,7 +67,6 @@ import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; import org.rocksdb.RocksIterator; -import org.rocksdb.SidePluginRepo; import org.rocksdb.SstFileManager; import org.rocksdb.TableFormatConfig; import org.rocksdb.WriteBatch; @@ -389,18 +391,58 @@ private static OpenedRocksDB openRocksDB(HugeConfig config, String dataPath, * Don't merge old CFs, we expect a clear DB when using this one */ RocksDB rocksdb = null; - SidePluginRepo repo = null; - // use rocksdb - if (StringUtils.isEmpty(optionPath)) { - LOG.info("Use rocksdb open default CF"); + Object repo = null; + + boolean useTopling = false; + if (!StringUtils.isEmpty(optionPath)) { + try { + Class.forName("org.rocksdb.SidePluginRepo"); + useTopling = true; + LOG.info("SidePluginRepo found. Will attempt to open default CF RocksDB using " + + "Topling."); + } catch (ClassNotFoundException e) { + LOG.warn("SidePluginRepo not found, even though 'optionPath' was provided. " + + "Falling back to the standard RocksDB default CF opening method. " + + "The configuration in '{}' will be ignored.", optionPath); + } + } + + if (useTopling) { + try { + // Dynamically load the SidePluginRepo class by its name at runtime. + Class sidePluginRepoClass = Class.forName("org.rocksdb.SidePluginRepo"); + + repo = sidePluginRepoClass.getConstructor().newInstance(); + String dbName = getDbName(dataPath); + Method putMethod = + sidePluginRepoClass.getMethod("put", String.class, DBOptions.class); + putMethod.invoke(repo, dbName, options); + Method importAutoFileMethod = + sidePluginRepoClass.getMethod("importAutoFile", String.class); + importAutoFileMethod.invoke(repo, optionPath); + + Method openDBMethod = sidePluginRepoClass.getMethod("openDB", String.class); + rocksdb = (RocksDB) openDBMethod.invoke(repo, converseOptionsToJsonString(dataPath, + null)); + } catch (ClassNotFoundException e) { + // CRITICAL: If the class is not found, the current rocksdbjni library does not + // include SidePluginRepo. + LOG.error( + "SidePluginRepo not found. This version of rocksdbjni does not support " + + "topling.", + e); + // Since the user provided an optionPath, the intent was to use a feature that is + // unavailable. Throwing an exception is the correct course of action. + throw new IllegalStateException( + "Topling features (SidePluginRepo) are required but not found in the " + + "rocksdbjni library.", + e); + } catch (InvocationTargetException | InstantiationException | IllegalAccessException | + NoSuchMethodException e) { + throw new RuntimeException(e); + } + } else { rocksdb = RocksDB.open(options, dataPath); - } else { // use topling - LOG.info("Use topling open default CF"); - repo = new SidePluginRepo(); - String dbName = getDbName(dataPath); - repo.put(dbName, options); - repo.importAutoFile(optionPath); - rocksdb = repo.openDB(converseOptionsToJsonString(dataPath, null)); } Map cfs = new ConcurrentHashMap<>(); @@ -437,18 +479,62 @@ private static OpenedRocksDB openRocksDB(HugeConfig config, List cfhs = new ArrayList<>(); RocksDB rocksdb = null; - SidePluginRepo repo = null; - // use rocksdb - if (StringUtils.isEmpty(optionPath)) { - LOG.info("Use rocksdb open multi CFs"); + Object repo = null; + + boolean useTopling = false; + if (!StringUtils.isEmpty(optionPath)) { + try { + Class.forName("org.rocksdb.SidePluginRepo"); + useTopling = true; + LOG.info("SidePluginRepo found. Will attempt to open multi CFs RocksDB using " + + "Topling plugin."); + } catch (ClassNotFoundException e) { + LOG.warn("SidePluginRepo not found, even though 'optionPath' was provided. " + + "Falling back to the standard RocksDB opening multi CFs method. " + + "The configuration in '{}' will be ignored.", optionPath); + } + } + + if (useTopling) { + try { + // Use reflection to check for the existence of the SidePluginRepo class at + // runtime. + Class sidePluginRepoClass = Class.forName("org.rocksdb.SidePluginRepo"); + + // Get the constructor and create a new instance. + Constructor constructor = sidePluginRepoClass.getConstructor(); + repo = constructor.newInstance(); + + String dbName = getDbName(dataPath); + + Method putMethod = + sidePluginRepoClass.getMethod("put", String.class, DBOptions.class); + putMethod.invoke(repo, dbName, options); + + Method importAutoFileMethod = + sidePluginRepoClass.getMethod("importAutoFile", String.class); + importAutoFileMethod.invoke(repo, optionPath); + + Method openDBMethod = + sidePluginRepoClass.getMethod("openDB", String.class, List.class); + rocksdb = (RocksDB) openDBMethod.invoke(repo, + converseOptionsToJsonString(dataPath, cfs), + cfhs); + + LOG.info("Successfully opened DB with SidePluginRepo."); + } catch (ClassNotFoundException e) { + // In this case, this exception should not occur + throw new IllegalStateException( + "Topling features (SidePluginRepo) are required but not found in the " + + "rocksdbjni library.", + e); + } catch (InvocationTargetException | InstantiationException | NoSuchMethodException | + IllegalAccessException e) { + throw new RuntimeException(e); + } + } else { + // use rocksdb rocksdb = RocksDB.open(options, dataPath, cfds, cfhs); - } else { // use topling - LOG.info("Use topling open multi CFs"); - repo = new SidePluginRepo(); - String dbName = getDbName(dataPath); - repo.put(dbName, options); - repo.importAutoFile(optionPath); - rocksdb = repo.openDB(converseOptionsToJsonString(dataPath, cfs), cfhs); } E.checkState(cfhs.size() == cfs.size(), From 9ae925bd2d095cf91298a21ef40dab40957caa6d Mon Sep 17 00:00:00 2001 From: gydeng Date: Mon, 23 Jun 2025 08:22:20 +0000 Subject: [PATCH 04/51] fix(server): fix bugs when using reflection openDB --- .../backend/store/rocksdb/RocksDBStdSessions.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java index ccaed5de19..8018c04762 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java @@ -415,7 +415,7 @@ private static OpenedRocksDB openRocksDB(HugeConfig config, String dataPath, repo = sidePluginRepoClass.getConstructor().newInstance(); String dbName = getDbName(dataPath); Method putMethod = - sidePluginRepoClass.getMethod("put", String.class, DBOptions.class); + sidePluginRepoClass.getMethod("put", String.class, Options.class); putMethod.invoke(repo, dbName, options); Method importAutoFileMethod = sidePluginRepoClass.getMethod("importAutoFile", String.class); @@ -530,7 +530,12 @@ private static OpenedRocksDB openRocksDB(HugeConfig config, e); } catch (InvocationTargetException | InstantiationException | NoSuchMethodException | IllegalAccessException e) { - throw new RuntimeException(e); + Throwable rootCause = e; + + while (rootCause.getCause() != null && rootCause.getCause() != rootCause) { + rootCause = rootCause.getCause(); + } + throw new RocksDBException(rootCause.getMessage()); } } else { // use rocksdb From 448551a8d2228bd8d8a7fc446070943cb16a2f55 Mon Sep 17 00:00:00 2001 From: gydeng Date: Mon, 30 Jun 2025 11:26:48 +0000 Subject: [PATCH 05/51] feat: add some scripts to unzip and preload toplingdb from the jar package --- .github/configs/settings.xml | 7 ++ .../src/assembly/static/bin/init-store.sh | 2 + .../assembly/static/bin/preload-topling.sh | 70 +++++++++++++++++ .../assembly/static/bin/start-hugegraph.sh | 2 + .../conf/graphs/db_bench_community.yaml | 1 + .../src/assembly/travis/install-backend.sh | 3 + .../src/assembly/travis/install-rocksdb.sh | 75 +++++++++++++++++++ .../backend/store/rocksdb/RocksDBOptions.java | 2 +- 8 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh create mode 100644 hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh diff --git a/.github/configs/settings.xml b/.github/configs/settings.xml index 45c87139a9..6cc7dc435c 100644 --- a/.github/configs/settings.xml +++ b/.github/configs/settings.xml @@ -27,6 +27,13 @@ staged-releases https://repository.apache.org/content/groups/staging/ + + github + https://maven.pkg.github.com/hugegraph/toplingdb + + true + + diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/init-store.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/init-store.sh index cde2cff742..2030c2dfc2 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/init-store.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/init-store.sh @@ -52,6 +52,8 @@ if [[ $? -eq 0 && $JAVA_VERSION > "1.9" ]]; then DEFAULT_JAVA_OPTIONS="--add-exports=java.base/jdk.internal.reflect=ALL-UNNAMED" fi +source $BIN/preload-topling.sh + echo "Initializing HugeGraph Store..." CP=$(find "${LIB}" "${PLUGINS}" -name "*.jar" | tr "\n" ":") diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh new file mode 100644 index 0000000000..8f2112cda9 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +function abs_path() { + SOURCE="${BASH_SOURCE[0]}" + while [[ -h "$SOURCE" ]]; do + DIR="$(cd -P "$(dirname "$SOURCE")" && pwd)" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + done + cd -P "$(dirname "$SOURCE")" && pwd +} + +function extract_so_with_jar() { + local jar_file="$1" + local dest_dir="$2" + local abs_jar_path + local pipeline_status + + if [ ! -f "$jar_file" ]; then + echo "'$jar_file' Not Exist" >&2 + fi + + if ! mkdir -p "$dest_dir"; then + echo "Cannot mkdir '$dest_dir'" >&2 + fi + + if [[ "$jar_file" == /* ]]; then + abs_jar_path="$jar_file" + else + abs_jar_path="$(pwd)/$jar_file" + fi + + (cd "$dest_dir" && jar tf "$abs_jar_path" | grep '\.so$' | xargs jar xf "$abs_jar_path") + pipeline_status=$? + + if [ $pipeline_status -ne 0 ]; then + echo "(Error: $pipeline_status)" >&2 + fi +} + +function preload_toplingdb() { + local jar_file=$(find $LIB -name "rocksdbjni*.jar") + local dest_dir="$TOP/library" + + extract_so_with_jar $jar_file $dest_dir + export LD_LIBRARY_PATH=$dest_dir:$LD_LIBRARY_PATH + export LD_PRELOAD=libjemalloc.so:librocksdbjni-linux64.so +} + +BIN=$(abs_path) +TOP="$(cd "$BIN"/../ && pwd)" +LIB="$TOP/lib" + +preload_toplingdb diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh index 30c2764989..72519440f4 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh @@ -99,6 +99,8 @@ if [[ $PRELOAD == "true" ]]; then sed -i -e '/registerBackends/d; /serverStarted/d' "${SCRIPTS}/${EXAMPLE_SCRIPT}" fi +source $BIN/preload-topling.sh + # TODO: show the output message in hugegraph-server.sh when start the server if [[ $DAEMON == "true" ]]; then echo "Starting HugeGraphServer in daemon mode..." diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/db_bench_community.yaml b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/db_bench_community.yaml index 86e74a19cf..74c6fb2321 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/db_bench_community.yaml +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/db_bench_community.yaml @@ -115,6 +115,7 @@ CFOptions: level0_slowdown_writes_trigger: 20 level0_stop_writes_trigger: 36 level0_file_num_compaction_trigger: 4 + merge_operator: uint64add # support merge compression_per_level: - kNoCompression - kNoCompression diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/install-backend.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-backend.sh index 917fc742f1..8974763c83 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/install-backend.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-backend.sh @@ -54,6 +54,9 @@ case $BACKEND in hstore) "$TRAVIS_DIR"/install-hstore.sh ;; + rocksdb) + source "$TRAVIS_DIR"/install-rocksdb.sh + ;; *) # don't need to install for other backends ;; diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh new file mode 100644 index 0000000000..8d1087bda2 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +function abs_path() { + SOURCE="${BASH_SOURCE[0]}" + while [[ -h "$SOURCE" ]]; do + DIR="$(cd -P "$(dirname "$SOURCE")" && pwd)" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + done + cd -P "$(dirname "$SOURCE")" && pwd +} + +function extract_so_with_jar() { + local jar_file="$1" + local dest_dir="$2" + local abs_jar_path + local pipeline_status + + if [ ! -f "$jar_file" ]; then + echo "'$jar_file' Not Exist" >&2 + fi + + if ! mkdir -p "$dest_dir"; then + echo "Cannot mkdir '$dest_dir'" >&2 + fi + + if [[ "$jar_file" == /* ]]; then + abs_jar_path="$jar_file" + else + abs_jar_path="$(pwd)/$jar_file" + fi + + (cd "$dest_dir" && jar tf "$abs_jar_path" | grep '\.so$' | xargs jar xf "$abs_jar_path") + pipeline_status=$? + + if [ $pipeline_status -ne 0 ]; then + echo "(Error: $pipeline_status)" >&2 + fi +} + +function preload_toplingdb() { + local jar_file=$(find $LIB -name "rocksdbjni*.jar") + local dest_dir="$TOP/library" + + extract_so_with_jar $jar_file $dest_dir + export LD_LIBRARY_PATH=$dest_dir:$LD_LIBRARY_PATH + export LD_PRELOAD=libjemalloc.so:librocksdbjni-linux64.so +} + +VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) +TOP=hugegraph-server/apache-hugegraph-server-incubating-$VERSION/ +BIN="$TOP/bin" +LIB="$TOP/lib" +CONF="$TOP/conf" +DB_CONF="$CONF/graphs/db_bench_community.yaml" + +preload_toplingdb + +cp $DB_CONF . diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java index 67052373a5..de3a49c040 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java @@ -91,7 +91,7 @@ public static synchronized RocksDBOptions instance() { "rocksdb.option_path", "The path of the JSON file for configuring RocksDB options.", null, - "" + "db_bench_community.yaml" ); public static final ConfigOption SST_PATH = From 8ac2fdc95490fc1141df82b174471f62c75e5bd6 Mon Sep 17 00:00:00 2001 From: gydeng Date: Sat, 5 Jul 2025 09:34:07 +0000 Subject: [PATCH 06/51] chore: fix CI bugs --- .github/workflows/server-ci.yml | 10 +++++----- .../src/assembly/static/bin/preload-topling.sh | 3 +++ .../src/assembly/travis/install-rocksdb.sh | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index f91bd809a3..d92526a042 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -13,7 +13,7 @@ jobs: # TODO: we need test & replace it to ubuntu-24.04 or ubuntu-latest runs-on: ubuntu-22.04 env: - USE_STAGE: 'false' # Whether to include the stage repository. + USE_STAGE: 'true' # Whether to include the stage repository. TRAVIS_DIR: hugegraph-server/hugegraph-dist/src/assembly/travis REPORT_DIR: target/site/jacoco BACKEND: ${{ matrix.BACKEND }} @@ -43,10 +43,6 @@ jobs: java-version: '8' distribution: 'zulu' - - name: Prepare backend environment - run: | - $TRAVIS_DIR/install-backend.sh $BACKEND && jps -l - - name: Install Java ${{ matrix.JAVA_VERSION }} uses: actions/setup-java@v4 with: @@ -70,6 +66,10 @@ jobs: run: | mvn clean compile -U -Dmaven.javadoc.skip=true -ntp + - name: Prepare backend environment + run: | + $TRAVIS_DIR/install-backend.sh $BACKEND && jps -l + - name: Run unit test run: | $TRAVIS_DIR/run-unit-test.sh $BACKEND diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh index 8f2112cda9..0436e3d870 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh @@ -66,5 +66,8 @@ function preload_toplingdb() { BIN=$(abs_path) TOP="$(cd "$BIN"/../ && pwd)" LIB="$TOP/lib" +YAML="$TOP/conf/graphs/db_bench_community.yaml" preload_toplingdb + +cp $YAML . diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh index 8d1087bda2..7905d4519f 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh @@ -64,7 +64,7 @@ function preload_toplingdb() { } VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) -TOP=hugegraph-server/apache-hugegraph-server-incubating-$VERSION/ +TOP=hugegraph-server/apache-hugegraph-server-incubating-$VERSION BIN="$TOP/bin" LIB="$TOP/lib" CONF="$TOP/conf" From 90cb28f58edc0c1b3782cb081758464cbbbee30b Mon Sep 17 00:00:00 2001 From: gydeng Date: Sat, 5 Jul 2025 09:42:18 +0000 Subject: [PATCH 07/51] chore: change env.USE_STAGE to true --- .github/workflows/check-dependencies.yml | 2 +- .github/workflows/cluster-test-ci.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/commons-ci.yml | 4 ++-- .github/workflows/licence-checker.yml | 2 +- .github/workflows/pd-store-ci.yml | 6 +++--- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/check-dependencies.yml b/.github/workflows/check-dependencies.yml index fc6d969e37..68f8c0e0c9 100644 --- a/.github/workflows/check-dependencies.yml +++ b/.github/workflows/check-dependencies.yml @@ -13,7 +13,7 @@ jobs: dependency-check: runs-on: ubuntu-latest env: - USE_STAGE: 'false' # Whether to include the stage repository. + USE_STAGE: 'true' # Whether to include the stage repository. SCRIPT_DEPENDENCY: install-dist/scripts/dependency steps: - name: Checkout source diff --git a/.github/workflows/cluster-test-ci.yml b/.github/workflows/cluster-test-ci.yml index 3ef269e878..9f5f254629 100644 --- a/.github/workflows/cluster-test-ci.yml +++ b/.github/workflows/cluster-test-ci.yml @@ -12,7 +12,7 @@ jobs: cluster-test: runs-on: ubuntu-latest env: - USE_STAGE: 'false' # Whether to include the stage repository. + USE_STAGE: 'true' # Whether to include the stage repository. steps: - name: Install JDK 11 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 538eb7f98c..9165bfda94 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -12,7 +12,7 @@ on: jobs: analyze: env: - USE_STAGE: 'false' # Whether to include the stage repository. + USE_STAGE: 'true' # Whether to include the stage repository. name: Analyze runs-on: ubuntu-latest permissions: diff --git a/.github/workflows/commons-ci.yml b/.github/workflows/commons-ci.yml index 5311ebeee0..1742701eaa 100644 --- a/.github/workflows/commons-ci.yml +++ b/.github/workflows/commons-ci.yml @@ -13,12 +13,12 @@ jobs: build-commons: runs-on: ubuntu-latest env: - USE_STAGE: 'false' # Whether to include the stage repository. + USE_STAGE: 'true' # Whether to include the stage repository. strategy: fail-fast: false matrix: - JAVA_VERSION: ['11'] + JAVA_VERSION: [ '11' ] steps: - name: Install JDK ${{ matrix.JAVA_VERSION }} diff --git a/.github/workflows/licence-checker.yml b/.github/workflows/licence-checker.yml index a6e6990a64..11559e7060 100644 --- a/.github/workflows/licence-checker.yml +++ b/.github/workflows/licence-checker.yml @@ -11,7 +11,7 @@ jobs: check-license: runs-on: ubuntu-latest env: - USE_STAGE: 'false' # Whether to include the stage repository. + USE_STAGE: 'true' # Whether to include the stage repository. steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/pd-store-ci.yml b/.github/workflows/pd-store-ci.yml index 6915e22eb2..916fb50303 100644 --- a/.github/workflows/pd-store-ci.yml +++ b/.github/workflows/pd-store-ci.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest env: # TODO: avoid duplicated env setup in pd & store - USE_STAGE: 'false' # Whether to include the stage repository. + USE_STAGE: 'true' # Whether to include the stage repository. # TODO: remove outdated env TRAVIS_DIR: hugegraph-server/hugegraph-dist/src/assembly/travis REPORT_DIR: target/site/jacoco @@ -79,7 +79,7 @@ jobs: # TODO: avoid duplicated env setup runs-on: ubuntu-latest env: - USE_STAGE: 'false' # Whether to include the stage repository. + USE_STAGE: 'true' # Whether to include the stage repository. # TODO: remove outdated env TRAVIS_DIR: hugegraph-server/hugegraph-dist/src/assembly/travis REPORT_DIR: target/site/jacoco @@ -151,7 +151,7 @@ jobs: # TODO: avoid duplicated env setup runs-on: ubuntu-latest env: - USE_STAGE: 'false' # Whether to include the stage repository. + USE_STAGE: 'true' # Whether to include the stage repository. TRAVIS_DIR: hugegraph-server/hugegraph-dist/src/assembly/travis REPORT_DIR: target/site/jacoco BACKEND: hstore From 31097ed641f2e649f00b4689135245e242515078 Mon Sep 17 00:00:00 2001 From: gydeng Date: Sun, 6 Jul 2025 02:34:14 +0000 Subject: [PATCH 08/51] chore: try to modify the permission to solve the 401 problem --- .github/workflows/server-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index d92526a042..42e6cb46b0 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -12,6 +12,9 @@ jobs: build-server: # TODO: we need test & replace it to ubuntu-24.04 or ubuntu-latest runs-on: ubuntu-22.04 + permissions: + contents: read + packages: read env: USE_STAGE: 'true' # Whether to include the stage repository. TRAVIS_DIR: hugegraph-server/hugegraph-dist/src/assembly/travis From 2fef639a51032b10bf6cfe32f14bf5716437d3cd Mon Sep 17 00:00:00 2001 From: gydeng Date: Tue, 8 Jul 2025 09:26:03 +0000 Subject: [PATCH 09/51] fix(server): solve the bug of importing topling package --- .github/workflows/check-dependencies.yml | 2 ++ .github/workflows/cluster-test-ci.yml | 2 ++ .github/workflows/codeql-analysis.yml | 2 ++ .github/workflows/commons-ci.yml | 2 ++ .github/workflows/licence-checker.yml | 2 ++ .github/workflows/pd-store-ci.yml | 6 ++++++ .github/workflows/server-ci.yml | 12 ++++++++++++ .../src/assembly/travis/install-backend.sh | 3 --- .../src/assembly/travis/install-rocksdb.sh | 16 ++++++++-------- .../src/assembly/travis/run-api-test.sh | 5 +++++ .../src/assembly/travis/run-core-test.sh | 6 ++++++ .../backend/store/rocksdb/RocksDBOptions.java | 2 +- 12 files changed, 48 insertions(+), 12 deletions(-) mode change 100644 => 100755 hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh diff --git a/.github/workflows/check-dependencies.yml b/.github/workflows/check-dependencies.yml index 68f8c0e0c9..5f04395504 100644 --- a/.github/workflows/check-dependencies.yml +++ b/.github/workflows/check-dependencies.yml @@ -15,6 +15,8 @@ jobs: env: USE_STAGE: 'true' # Whether to include the stage repository. SCRIPT_DEPENDENCY: install-dist/scripts/dependency + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ACTOR: ${{ github.actor }} steps: - name: Checkout source uses: actions/checkout@v4 diff --git a/.github/workflows/cluster-test-ci.yml b/.github/workflows/cluster-test-ci.yml index 9f5f254629..0537f31a6a 100644 --- a/.github/workflows/cluster-test-ci.yml +++ b/.github/workflows/cluster-test-ci.yml @@ -13,6 +13,8 @@ jobs: runs-on: ubuntu-latest env: USE_STAGE: 'true' # Whether to include the stage repository. + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ACTOR: ${{ github.actor }} steps: - name: Install JDK 11 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 9165bfda94..7db451adda 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -13,6 +13,8 @@ jobs: analyze: env: USE_STAGE: 'true' # Whether to include the stage repository. + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ACTOR: ${{ github.actor }} name: Analyze runs-on: ubuntu-latest permissions: diff --git a/.github/workflows/commons-ci.yml b/.github/workflows/commons-ci.yml index 1742701eaa..3bacaf9816 100644 --- a/.github/workflows/commons-ci.yml +++ b/.github/workflows/commons-ci.yml @@ -14,6 +14,8 @@ jobs: runs-on: ubuntu-latest env: USE_STAGE: 'true' # Whether to include the stage repository. + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ACTOR: ${{ github.actor }} strategy: fail-fast: false diff --git a/.github/workflows/licence-checker.yml b/.github/workflows/licence-checker.yml index 11559e7060..e1b8ee9f93 100644 --- a/.github/workflows/licence-checker.yml +++ b/.github/workflows/licence-checker.yml @@ -12,6 +12,8 @@ jobs: runs-on: ubuntu-latest env: USE_STAGE: 'true' # Whether to include the stage repository. + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ACTOR: ${{ github.actor }} steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/pd-store-ci.yml b/.github/workflows/pd-store-ci.yml index 916fb50303..c7ecf62fe0 100644 --- a/.github/workflows/pd-store-ci.yml +++ b/.github/workflows/pd-store-ci.yml @@ -18,6 +18,8 @@ jobs: # TODO: remove outdated env TRAVIS_DIR: hugegraph-server/hugegraph-dist/src/assembly/travis REPORT_DIR: target/site/jacoco + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ACTOR: ${{ github.actor }} steps: - name: Install JDK 11 @@ -83,6 +85,8 @@ jobs: # TODO: remove outdated env TRAVIS_DIR: hugegraph-server/hugegraph-dist/src/assembly/travis REPORT_DIR: target/site/jacoco + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ACTOR: ${{ github.actor }} steps: - name: Install JDK 11 @@ -156,6 +160,8 @@ jobs: REPORT_DIR: target/site/jacoco BACKEND: hstore RELEASE_BRANCH: ${{ startsWith(github.ref_name, 'release-') || startsWith(github.ref_name, 'test-') || startsWith(github.base_ref, 'release-') }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ACTOR: ${{ github.actor }} steps: - name: Install JDK 11 diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index 42e6cb46b0..08eee3b0d1 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -26,6 +26,8 @@ jobs: TARGET_BRANCH_NAME: ${{ github.base_ref != '' && github.base_ref || github.ref_name }} RELEASE_BRANCH: ${{ startsWith(github.ref_name, 'release-') || startsWith(github.ref_name, 'test-') }} RAFT_MODE: ${{ startsWith(github.head_ref, 'test') || startsWith(github.head_ref, 'raft') }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ACTOR: ${{ github.actor }} strategy: fail-fast: false @@ -52,6 +54,11 @@ jobs: java-version: ${{ matrix.JAVA_VERSION }} distribution: 'zulu' + - name: Install deps + run: | + sudo apt-get update -y + sudo apt-get install -y liburing-dev libaio-dev libjemalloc-dev + - name: Cache Maven packages uses: actions/cache@v4 with: @@ -69,6 +76,11 @@ jobs: run: | mvn clean compile -U -Dmaven.javadoc.skip=true -ntp + - name: Package + run: | + mvn package -Dmaven.test.skip=true -ntp + source $TRAVIS_DIR/install-rocksdb.sh + - name: Prepare backend environment run: | $TRAVIS_DIR/install-backend.sh $BACKEND && jps -l diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/install-backend.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-backend.sh index 8974763c83..917fc742f1 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/install-backend.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-backend.sh @@ -54,9 +54,6 @@ case $BACKEND in hstore) "$TRAVIS_DIR"/install-hstore.sh ;; - rocksdb) - source "$TRAVIS_DIR"/install-rocksdb.sh - ;; *) # don't need to install for other backends ;; diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh old mode 100644 new mode 100755 index 7905d4519f..ac7b8dc16f --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh @@ -56,7 +56,7 @@ function extract_so_with_jar() { function preload_toplingdb() { local jar_file=$(find $LIB -name "rocksdbjni*.jar") - local dest_dir="$TOP/library" + local dest_dir="$(pwd)"/$LIBRARY extract_so_with_jar $jar_file $dest_dir export LD_LIBRARY_PATH=$dest_dir:$LD_LIBRARY_PATH @@ -64,12 +64,12 @@ function preload_toplingdb() { } VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) -TOP=hugegraph-server/apache-hugegraph-server-incubating-$VERSION -BIN="$TOP/bin" -LIB="$TOP/lib" -CONF="$TOP/conf" -DB_CONF="$CONF/graphs/db_bench_community.yaml" +SERVER_DIR=hugegraph-server/apache-hugegraph-server-incubating-$VERSION +BIN=$SERVER_DIR/bin +LIB=$SERVER_DIR/lib +CONF=$SERVER_DIR/conf +DB_CONF=$CONF/graphs/db_bench_community.yaml +LIBRARY=$SERVER_DIR/library +GITHUB="https://github.com" preload_toplingdb - -cp $DB_CONF . diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test.sh index 2a3c2c35ee..ba7f7be7eb 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test.sh @@ -31,6 +31,11 @@ JACOCO_PORT=36320 mvn package -Dmaven.test.skip=true -ntp +# install rocksdb +if [[ "$BACKEND" == "rocksdb" ]]; then + source $TRAVIS_DIR/install-rocksdb.sh +fi + # add mysql dependency wget -P $SERVER_DIR/lib/ https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.28/mysql-connector-java-8.0.28.jar diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-core-test.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-core-test.sh index a95d2f0806..e719a79688 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-core-test.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-core-test.sh @@ -18,5 +18,11 @@ set -ev BACKEND=$1 +TRAVIS_DIR=$(dirname $0) + +# install rocksdb +if [[ "$BACKEND" == "rocksdb" ]]; then + source $TRAVIS_DIR/install-rocksdb.sh +fi mvn test -pl hugegraph-server/hugegraph-test -am -P core-test,$BACKEND diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java index de3a49c040..67052373a5 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java @@ -91,7 +91,7 @@ public static synchronized RocksDBOptions instance() { "rocksdb.option_path", "The path of the JSON file for configuring RocksDB options.", null, - "db_bench_community.yaml" + "" ); public static final ConfigOption SST_PATH = From cf4dbe76c858ec5e64966bb910716901e2f6325d Mon Sep 17 00:00:00 2001 From: gydeng Date: Thu, 10 Jul 2025 05:05:59 +0000 Subject: [PATCH 10/51] fix(CI): eliminate pollution of environment variables --- .../src/assembly/static/bin/preload-topling.sh | 5 +++-- .../src/assembly/travis/install-rocksdb.sh | 9 ++++----- .../hugegraph-dist/src/assembly/travis/run-api-test.sh | 4 +--- .../hugegraph-dist/src/assembly/travis/run-core-test.sh | 4 +--- .../hugegraph-dist/src/assembly/travis/run-unit-test.sh | 3 +++ 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh index 0436e3d870..9a7c4c415b 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh @@ -46,11 +46,12 @@ function extract_so_with_jar() { abs_jar_path="$(pwd)/$jar_file" fi - (cd "$dest_dir" && jar tf "$abs_jar_path" | grep '\.so$' | xargs jar xf "$abs_jar_path") + unzip -j -o "$abs_jar_path" "*.so" -d "$dest_dir" > /dev/null pipeline_status=$? if [ $pipeline_status -ne 0 ]; then - echo "(Error: $pipeline_status)" >&2 + # unzip provides specific exit codes that can be more descriptive. + echo "Error: Failed to extract .so files with unzip (Exit Code: $pipeline_status)" >&2 fi } diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh index ac7b8dc16f..6ebd5f474d 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh @@ -46,11 +46,12 @@ function extract_so_with_jar() { abs_jar_path="$(pwd)/$jar_file" fi - (cd "$dest_dir" && jar tf "$abs_jar_path" | grep '\.so$' | xargs jar xf "$abs_jar_path") + unzip -j -o "$abs_jar_path" "*.so" -d "$dest_dir" > /dev/null pipeline_status=$? if [ $pipeline_status -ne 0 ]; then - echo "(Error: $pipeline_status)" >&2 + # unzip provides specific exit codes that can be more descriptive. + echo "Error: Failed to extract .so files with unzip (Exit Code: $pipeline_status)" >&2 fi } @@ -65,10 +66,8 @@ function preload_toplingdb() { VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) SERVER_DIR=hugegraph-server/apache-hugegraph-server-incubating-$VERSION -BIN=$SERVER_DIR/bin LIB=$SERVER_DIR/lib -CONF=$SERVER_DIR/conf -DB_CONF=$CONF/graphs/db_bench_community.yaml +DB_CONF=$SERVER_DIR/conf/graphs/db_bench_community.yaml LIBRARY=$SERVER_DIR/library GITHUB="https://github.com" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test.sh index ba7f7be7eb..c0bca01feb 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test.sh @@ -32,9 +32,7 @@ JACOCO_PORT=36320 mvn package -Dmaven.test.skip=true -ntp # install rocksdb -if [[ "$BACKEND" == "rocksdb" ]]; then - source $TRAVIS_DIR/install-rocksdb.sh -fi +source $TRAVIS_DIR/install-rocksdb.sh # add mysql dependency wget -P $SERVER_DIR/lib/ https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.28/mysql-connector-java-8.0.28.jar diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-core-test.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-core-test.sh index e719a79688..6ced5d93b0 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-core-test.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-core-test.sh @@ -21,8 +21,6 @@ BACKEND=$1 TRAVIS_DIR=$(dirname $0) # install rocksdb -if [[ "$BACKEND" == "rocksdb" ]]; then - source $TRAVIS_DIR/install-rocksdb.sh -fi +source $TRAVIS_DIR/install-rocksdb.sh mvn test -pl hugegraph-server/hugegraph-test -am -P core-test,$BACKEND diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-unit-test.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-unit-test.sh index 5fe9b476b3..6b1b22225a 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-unit-test.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-unit-test.sh @@ -19,6 +19,9 @@ set -ev BACKEND=$1 +# install rocksdb +source $TRAVIS_DIR/install-rocksdb.sh + if [[ "$BACKEND" == "memory" ]]; then mvn test -pl hugegraph-server/hugegraph-test -am -P unit-test fi From ccaa2b912f29038a2d05ea6aa422bf9963c6ab38 Mon Sep 17 00:00:00 2001 From: gydeng Date: Sun, 13 Jul 2025 02:05:41 +0000 Subject: [PATCH 11/51] install deps in hstore-ci --- .github/workflows/pd-store-ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/pd-store-ci.yml b/.github/workflows/pd-store-ci.yml index c7ecf62fe0..ca4eff81e3 100644 --- a/.github/workflows/pd-store-ci.yml +++ b/.github/workflows/pd-store-ci.yml @@ -182,6 +182,11 @@ jobs: with: fetch-depth: 2 + - name: Install deps + run: | + sudo apt-get update -y + sudo apt-get install -y liburing-dev libaio-dev libjemalloc-dev + - name: use staged maven repo settings if: ${{ env.USE_STAGE == 'true' }} run: | From cfee29baca3dc8928cc9821914eecb13a4e9592d Mon Sep 17 00:00:00 2001 From: gydeng Date: Sun, 13 Jul 2025 02:47:00 +0000 Subject: [PATCH 12/51] add compatibility for libaio.so.1 for ubuntu-24.04 --- .../src/assembly/static/bin/preload-topling.sh | 17 +++++++++++++++++ .../src/assembly/travis/install-rocksdb.sh | 17 +++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh index 9a7c4c415b..db3a8c2b81 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh @@ -59,6 +59,23 @@ function preload_toplingdb() { local jar_file=$(find $LIB -name "rocksdbjni*.jar") local dest_dir="$TOP/library" + # Check for Ubuntu 24.04+ and create a symlink for libaio if needed. + # This is a workaround for software expecting the old libaio.so.1 name, + # as it was renamed to libaio.so.1t64 in the new release. + # https://askubuntu.com/questions/1512196/libaio1-on-noble/1516639#1516639 + if [ -f /etc/os-release ]; then + # Source the os-release file to get ID and VERSION_ID variables + . /etc/os-release + # Use dpkg to reliably compare version numbers + if [ "$ID" = "ubuntu" ] && dpkg --compare-versions "$VERSION_ID" "ge" "24.04"; then + local libaio_link_target="/usr/lib/x86_64-linux-gnu/libaio.so.1" + if [ ! -e "$libaio_link_target" ]; then + echo "Ubuntu 24.04 or newer detected. Creating compatibility symlink for libaio." + sudo ln -s /usr/lib/x86_64-linux-gnu/libaio.so.1t64 "$libaio_link_target" + fi + fi + fi + extract_so_with_jar $jar_file $dest_dir export LD_LIBRARY_PATH=$dest_dir:$LD_LIBRARY_PATH export LD_PRELOAD=libjemalloc.so:librocksdbjni-linux64.so diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh index 6ebd5f474d..53b6217609 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh @@ -59,6 +59,23 @@ function preload_toplingdb() { local jar_file=$(find $LIB -name "rocksdbjni*.jar") local dest_dir="$(pwd)"/$LIBRARY + # Check for Ubuntu 24.04+ and create a symlink for libaio if needed. + # This is a workaround for software expecting the old libaio.so.1 name, + # as it was renamed to libaio.so.1t64 in the new release. + # https://askubuntu.com/questions/1512196/libaio1-on-noble/1516639#1516639 + if [ -f /etc/os-release ]; then + # Source the os-release file to get ID and VERSION_ID variables + . /etc/os-release + # Use dpkg to reliably compare version numbers + if [ "$ID" = "ubuntu" ] && dpkg --compare-versions "$VERSION_ID" "ge" "24.04"; then + local libaio_link_target="/usr/lib/x86_64-linux-gnu/libaio.so.1" + if [ ! -e "$libaio_link_target" ]; then + echo "Ubuntu 24.04 or newer detected. Creating compatibility symlink for libaio." + sudo ln -s /usr/lib/x86_64-linux-gnu/libaio.so.1t64 "$libaio_link_target" + fi + fi + fi + extract_so_with_jar $jar_file $dest_dir export LD_LIBRARY_PATH=$dest_dir:$LD_LIBRARY_PATH export LD_PRELOAD=libjemalloc.so:librocksdbjni-linux64.so From 461fee3ef2bb223f09e201b57ccce954065f9bf0 Mon Sep 17 00:00:00 2001 From: gydeng Date: Mon, 14 Jul 2025 07:37:40 +0000 Subject: [PATCH 13/51] chore(server): modify the configuration file to keep it consistent with the HG default configuration --- .../conf/graphs/db_bench_community.yaml | 35 +++++++++++-------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/db_bench_community.yaml b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/db_bench_community.yaml index 74c6fb2321..aeba6ec2cf 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/db_bench_community.yaml +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/db_bench_community.yaml @@ -101,36 +101,43 @@ TableFactory: level_writers: [ bb, bb, bb, bb, bb, bb ] CFOptions: default: - max_write_buffer_number: 4 + max_write_buffer_number: 6 memtable_factory: "${cspp}" - write_buffer_size: 8M + write_buffer_size: 128M # set target_file_size_base as small as 512K is to make many SST files, # thus key prefix cache can present efficiency - target_file_size_base: 512K + target_file_size_base: 64M target_file_size_multiplier: 1 table_factory: dispatch - max_bytes_for_level_base: 4M - max_bytes_for_level_multiplier: 4 + max_bytes_for_level_base: 512M + max_bytes_for_level_multiplier: 10 level_compaction_dynamic_level_bytes: false level0_slowdown_writes_trigger: 20 level0_stop_writes_trigger: 36 - level0_file_num_compaction_trigger: 4 + level0_file_num_compaction_trigger: 2 merge_operator: uint64add # support merge + level_compaction_dynamic_file_size: true + optimize_filters_for_hits: true + allow_merge_memtables: true + min_write_buffer_number_to_merge: 2 compression_per_level: - kNoCompression - kNoCompression - - kZSTD - - kZSTD - - kZSTD - - kZSTD - - kZSTD + - kSnappyCompression + - kSnappyCompression + - kSnappyCompression + - kSnappyCompression + - kSnappyCompression DBOptions: dbo: create_if_missing: true create_missing_column_families: false # this is important, must be false to hugegraph - max_background_compactions: 13 - max_subcompactions: 1 - max_level1_subcompactions: 7 + max_background_compactions: -1 + max_subcompactions: 4 + max_level1_subcompactions: 0 inplace_update_support: false WAL_size_limit_MB: 0 statistics: "${stat}" + max_manifest_file_size: 100M + max_background_jobs: 8 + compaction_readahead_size: 0 From 7cfa2eedae57322a941b6f722bedacbfa090e7e1 Mon Sep 17 00:00:00 2001 From: gydeng Date: Tue, 2 Sep 2025 06:49:44 +0000 Subject: [PATCH 14/51] feat(server): add the configuration open_http to start the Topling HTTP Server --- .../assembly/static/bin/preload-topling.sh | 50 +++++++++++++++-- .../static/conf/graphs/hugegraph.properties | 3 +- ...bench_community.yaml => rocksdb_plus.yaml} | 15 +++++- .../backend/store/rocksdb/RocksDBOptions.java | 8 +++ .../store/rocksdb/RocksDBStdSessions.java | 53 ++++++++++++++----- .../backend/store/rocksdb/RocksDBStore.java | 22 ++++---- .../store/rocksdbsst/RocksDBSstStore.java | 3 +- .../unit/rocksdb/BaseRocksDBUnitTest.java | 2 +- .../unit/rocksdb/RocksDBSessionsTest.java | 2 +- 9 files changed, 127 insertions(+), 31 deletions(-) rename hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/{db_bench_community.yaml => rocksdb_plus.yaml} (92%) diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh index db3a8c2b81..e60db9e6e7 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh @@ -55,6 +55,48 @@ function extract_so_with_jar() { fi } +function extract_html_css_from_jar() { + local jar_file="$1" + local dest_dir="$2" + local abs_jar_path + local pipeline_status + local resource_target="/dev/shm/rocksdb_resource" + + if [ ! -f "$jar_file" ]; then + echo "Error: JAR file '$jar_file' does not exist." >&2 + return 1 + fi + + if ! mkdir -p "$dest_dir"; then + echo "Error: Cannot create destination directory '$dest_dir'." >&2 + return 1 + fi + + if [[ "$jar_file" == /* ]]; then + abs_jar_path="$jar_file" + else + abs_jar_path="$(pwd)/$jar_file" + fi + + unzip -j -o "$abs_jar_path" "*.html" "*.css" -d "$dest_dir" > /dev/null + pipeline_status=$? + + if [ $pipeline_status -eq 11 ]; then + echo "Notice: No .html or .css files found in '$jar_file'." >&2 + return 0 + elif [ $pipeline_status -ne 0 ]; then + echo "Error: unzip failed with exit code $pipeline_status" >&2 + return $pipeline_status + fi + + if ! mkdir -p "$resource_target"; then + echo "Error: Cannot create target directory '$resource_target'." >&2 + return 1 + fi + + cp -f "$dest_dir"/*.html "$dest_dir"/*.css "$resource_target" 2>/dev/null || true +} + function preload_toplingdb() { local jar_file=$(find $LIB -name "rocksdbjni*.jar") local dest_dir="$TOP/library" @@ -79,13 +121,13 @@ function preload_toplingdb() { extract_so_with_jar $jar_file $dest_dir export LD_LIBRARY_PATH=$dest_dir:$LD_LIBRARY_PATH export LD_PRELOAD=libjemalloc.so:librocksdbjni-linux64.so + + extract_html_css_from_jar $jar_file $dest_dir } BIN=$(abs_path) TOP="$(cd "$BIN"/../ && pwd)" LIB="$TOP/lib" -YAML="$TOP/conf/graphs/db_bench_community.yaml" - -preload_toplingdb +YAML="$TOP/conf/graphs/rocksdb_plus.yaml" -cp $YAML . +preload_toplingdb \ No newline at end of file diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/hugegraph.properties b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/hugegraph.properties index 23617106b7..fe17100b91 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/hugegraph.properties +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/hugegraph.properties @@ -42,7 +42,8 @@ search.text_analyzer_mode=INDEX # rocksdb backend config #rocksdb.data_path=/path/to/disk #rocksdb.wal_path=/path/to/disk -#rocksdb.option_path=./db_bench_community.yaml +#rocksdb.option_path=./conf/graphs/rocksdb_plus.yaml +#rocksdb.open_http=true # hbase backend config #hbase.hosts=localhost diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/db_bench_community.yaml b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_plus.yaml similarity index 92% rename from hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/db_bench_community.yaml rename to hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_plus.yaml index aeba6ec2cf..989b8fefdf 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/db_bench_community.yaml +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_plus.yaml @@ -17,7 +17,7 @@ # common parameters http: # normally parent path of db path - document_root: /dev/shm/db_bench_community + document_root: /dev/shm/rocksdb_resource listening_ports: '2011' setenv: StrSimpleEnvNameNotOverwrite: StringValue @@ -71,14 +71,20 @@ MemTableRepFactory: cspp: class: cspp params: - mem_cap: 2G + mem_cap: 16G use_vm: false token_use_idle: true + chunk_size: 16K + convert_to_sst: kFileMmap + sync_sst_file: false skiplist: class: SkipList params: lookahead: 0 TableFactory: + cspp_memtab_sst: + class: CSPPMemTabTable + params: # empty params bb: class: BlockBasedTable params: @@ -89,6 +95,9 @@ TableFactory: metadata_block_size: 4K enable_index_compression: true block_cache: "${lru_cache}" + readers: + BlockBasedTable: bb + CSPPMemTabTable: cspp_memtab_sst block_cache_compressed: persistent_cache: filter_policy: @@ -98,6 +107,7 @@ TableFactory: default: bb readers: BlockBasedTable: bb + CSPPMemTabTable: cspp_memtab_sst level_writers: [ bb, bb, bb, bb, bb, bb ] CFOptions: default: @@ -141,3 +151,4 @@ DBOptions: max_manifest_file_size: 100M max_background_jobs: 8 compaction_readahead_size: 0 + memtable_as_log_index: true \ No newline at end of file diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java index 67052373a5..3819897285 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java @@ -94,6 +94,14 @@ public static synchronized RocksDBOptions instance() { "" ); + public static final ConfigOption OPEN_HTTP = + new ConfigOption<>( + "rocksdb.open_http", + "Whether to start Topling's HTTP service", + disallowEmpty(), + false + ); + public static final ConfigOption SST_PATH = new ConfigOption<>( "rocksdb.sst_path", diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java index 8018c04762..c305d48819 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java @@ -41,6 +41,7 @@ import org.apache.hugegraph.backend.store.BackendEntry.BackendColumn; import org.apache.hugegraph.backend.store.BackendEntry.BackendColumnIterator; import org.apache.hugegraph.backend.store.BackendEntryIterator; +import org.apache.hugegraph.backend.store.BackendStoreProvider; import org.apache.hugegraph.backend.store.rocksdb.RocksDBIteratorPool.ReusedRocksIterator; import org.apache.hugegraph.config.CoreOptions; import org.apache.hugegraph.config.HugeConfig; @@ -85,32 +86,39 @@ public class RocksDBStdSessions extends RocksDBSessions { private final String dataPath; private final String walPath; private final String optionPath; + private final Boolean openHttp; private volatile OpenedRocksDB rocksdb; private final AtomicInteger refCount; public RocksDBStdSessions(HugeConfig config, String database, String store, - String dataPath, String walPath, String optionPath) throws - RocksDBException { + String dataPath, String walPath, String optionPath, + Boolean openHttp) throws + RocksDBException { super(config, database, store); this.config = config; this.dataPath = dataPath; this.walPath = walPath; this.optionPath = optionPath; - this.rocksdb = RocksDBStdSessions.openRocksDB(config, dataPath, walPath, optionPath); + this.openHttp = openHttp; + this.rocksdb = RocksDBStdSessions.openRocksDB(config, dataPath, walPath, optionPath, + openHttp); this.refCount = new AtomicInteger(1); } public RocksDBStdSessions(HugeConfig config, String database, String store, String dataPath, String walPath, - List cfNames, String optionPath) throws RocksDBException { + List cfNames, String optionPath, Boolean openHttp) throws + RocksDBException { super(config, database, store); this.config = config; this.dataPath = dataPath; this.walPath = walPath; this.optionPath = optionPath; + this.openHttp = openHttp; this.rocksdb = - RocksDBStdSessions.openRocksDB(config, cfNames, dataPath, walPath, optionPath); + RocksDBStdSessions.openRocksDB(config, cfNames, dataPath, walPath, optionPath, + openHttp); this.refCount = new AtomicInteger(1); this.ingestExternalFile(); @@ -123,6 +131,7 @@ private RocksDBStdSessions(HugeConfig config, String database, String store, this.dataPath = origin.dataPath; this.walPath = origin.walPath; this.optionPath = origin.optionPath; + this.openHttp = origin.openHttp; this.rocksdb = origin.rocksdb; this.refCount = origin.refCount; this.refCount.incrementAndGet(); @@ -219,7 +228,8 @@ public void reloadRocksDB() throws RocksDBException { this.rocksdb.close(); } this.rocksdb = RocksDBStdSessions.openRocksDB(this.config, ImmutableList.of(), - this.dataPath, this.walPath, this.optionPath); + this.dataPath, this.walPath, + this.optionPath, this.openHttp); } @Override @@ -313,7 +323,8 @@ public String buildSnapshotPath(String snapshotPrefix) { public String hardLinkSnapshot(String snapshotPath) throws RocksDBException { String snapshotLinkPath = this.dataPath + "_temp"; try (OpenedRocksDB rocksdb = openRocksDB(this.config, ImmutableList.of(), - snapshotPath, null, this.optionPath)) { + snapshotPath, null, this.optionPath, + this.openHttp)) { rocksdb.createCheckpoint(snapshotLinkPath); } LOG.info("The snapshot {} has been hard linked to {}", snapshotPath, snapshotLinkPath); @@ -378,8 +389,9 @@ private void ingestExternalFile() throws RocksDBException { } private static OpenedRocksDB openRocksDB(HugeConfig config, String dataPath, - String walPath, String optionPath) throws - RocksDBException { + String walPath, String optionPath, + Boolean openHttp) throws + RocksDBException { // Init options Options options = new Options(); RocksDBStdSessions.initOptions(config, options, options, options, options); @@ -424,6 +436,14 @@ private static OpenedRocksDB openRocksDB(HugeConfig config, String dataPath, Method openDBMethod = sidePluginRepoClass.getMethod("openDB", String.class); rocksdb = (RocksDB) openDBMethod.invoke(repo, converseOptionsToJsonString(dataPath, null)); + + if (openHttp && BackendStoreProvider.GRAPH_STORE.equals(dbName)) { + Method openHttpMethod = + sidePluginRepoClass.getMethod("startHttpServer"); + openHttpMethod.invoke(repo); + LOG.info("Topling HTTP Server has been started according to the " + + "listening_ports specified in " + optionPath); + } } catch (ClassNotFoundException e) { // CRITICAL: If the class is not found, the current rocksdbjni library does not // include SidePluginRepo. @@ -451,8 +471,9 @@ private static OpenedRocksDB openRocksDB(HugeConfig config, String dataPath, private static OpenedRocksDB openRocksDB(HugeConfig config, List cfNames, String dataPath, - String walPath, String optionPath) throws - RocksDBException { + String walPath, String optionPath, + Boolean openHttp) throws + RocksDBException { // Old CFs should always be opened Set mergedCFs = RocksDBStdSessions.mergeOldCFs(dataPath, cfNames); @@ -521,6 +542,14 @@ private static OpenedRocksDB openRocksDB(HugeConfig config, converseOptionsToJsonString(dataPath, cfs), cfhs); + if (openHttp && BackendStoreProvider.GRAPH_STORE.equals(dbName)) { + Method openHttpMethod = + sidePluginRepoClass.getMethod("startHttpServer"); + openHttpMethod.invoke(repo); + LOG.info("Topling HTTP Server has been started according to the " + + "listening_ports specified in " + optionPath); + } + LOG.info("Successfully opened DB with SidePluginRepo."); } catch (ClassNotFoundException e) { // In this case, this exception should not occur @@ -601,7 +630,7 @@ private static String converseOptionsToJsonString(String dataPath, List } private static String getDbName(String dataPath) { - return dataPath.substring(dataPath.lastIndexOf("/") + 1); + return Paths.get(dataPath).getFileName().toString(); } public static void initOptions(HugeConfig conf, diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStore.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStore.java index e4a5aaa002..fb16044697 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStore.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStore.java @@ -240,7 +240,7 @@ public synchronized void open(HugeConfig config) { openedDisks.add(disk); List tables = this.tableNames(e.getKey()); futures.add(openPool.submit(() -> { - this.open(config, disk, disk, tables, null); + this.open(config, disk, disk, tables, null, false); })); } } @@ -302,16 +302,18 @@ protected RocksDBSessions open(HugeConfig config, List tableNames) { String dataPath = this.wrapPath(config.get(RocksDBOptions.DATA_PATH)); String walPath = this.wrapPath(config.get(RocksDBOptions.WAL_PATH)); String optionPath = config.get(RocksDBOptions.OPTION_PATH); - return this.open(config, dataPath, walPath, tableNames, optionPath); + Boolean openHttp = config.get(RocksDBOptions.OPEN_HTTP); + return this.open(config, dataPath, walPath, tableNames, optionPath, openHttp); } protected RocksDBSessions open(HugeConfig config, String dataPath, - String walPath, List tableNames, String optionPath) { + String walPath, List tableNames, String optionPath, + Boolean openHttp) { LOG.info("Opening RocksDB with data path: {}", dataPath); RocksDBSessions sessions = null; try { sessions = this.openSessionPool(config, dataPath, - walPath, tableNames, optionPath); + walPath, tableNames, optionPath, openHttp); } catch (RocksDBException e) { RocksDBSessions origin = this.dbs.get(dataPath); if (origin != null) { @@ -344,7 +346,8 @@ protected RocksDBSessions open(HugeConfig config, String dataPath, none = null; } try { - sessions = this.openSessionPool(config, dataPath, walPath, none, optionPath); + sessions = this.openSessionPool(config, dataPath, walPath, none, optionPath, + openHttp); } catch (RocksDBException e1) { e = e1; } @@ -373,14 +376,15 @@ protected RocksDBSessions open(HugeConfig config, String dataPath, protected RocksDBSessions openSessionPool(HugeConfig config, String dataPath, String walPath, - List tableNames, String optionPath) throws - RocksDBException { + List tableNames, String optionPath, + Boolean openHttp) throws + RocksDBException { if (tableNames == null) { return new RocksDBStdSessions(config, this.database, this.store, dataPath, walPath, - optionPath); + optionPath, openHttp); } else { return new RocksDBStdSessions(config, this.database, this.store, - dataPath, walPath, tableNames, optionPath); + dataPath, walPath, tableNames, optionPath, openHttp); } } diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdbsst/RocksDBSstStore.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdbsst/RocksDBSstStore.java index e824cb0ee6..9cab1b8ccb 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdbsst/RocksDBSstStore.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdbsst/RocksDBSstStore.java @@ -38,7 +38,8 @@ public RocksDBSstStore(final BackendStoreProvider provider, @Override protected RocksDBSessions openSessionPool(HugeConfig config, String dataPath, String walPath, - List tableNames, String optionPath) + List tableNames, String optionPath, + Boolean openHttp) throws RocksDBException { if (tableNames == null) { return new RocksDBSstSessions(config, this.database(), this.store(), dataPath); diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/BaseRocksDBUnitTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/BaseRocksDBUnitTest.java index 4658c686cd..65bde3edf9 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/BaseRocksDBUnitTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/BaseRocksDBUnitTest.java @@ -109,7 +109,7 @@ protected static long getLong(byte[] bytes) { private static RocksDBSessions open(String table) throws RocksDBException { HugeConfig config = FakeObjects.newConfig(); RocksDBSessions rocks = new RocksDBStdSessions(config, "db", "store", - DB_PATH, DB_PATH, null); + DB_PATH, DB_PATH, null, false); rocks.createTable(table); return rocks; } diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/RocksDBSessionsTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/RocksDBSessionsTest.java index 6327eb06d7..34c226e46d 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/RocksDBSessionsTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/RocksDBSessionsTest.java @@ -191,7 +191,7 @@ public void testIngestSst() throws RocksDBException { Assert.assertFalse(sstSessions.existsTable(TABLE2)); RocksDBSessions rocks = - new RocksDBStdSessions(config, "db", "store", sstPath, sstPath, null); + new RocksDBStdSessions(config, "db", "store", sstPath, sstPath, null, false); // Will ingest sst file of TABLE1 rocks.createTable(TABLE1); Assert.assertEquals(ImmutableList.of("1000"), From ac0319641474a3bbb101b4fa3009fc48510d0fea Mon Sep 17 00:00:00 2001 From: gydeng Date: Tue, 2 Sep 2025 08:12:13 +0000 Subject: [PATCH 15/51] chore(server): fix formatting issues in scripts --- .../hugegraph-dist/src/assembly/static/bin/preload-topling.sh | 2 +- .../src/assembly/static/conf/graphs/rocksdb_plus.yaml | 2 +- pom.xml | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh index e60db9e6e7..c743ab91bd 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh @@ -130,4 +130,4 @@ TOP="$(cd "$BIN"/../ && pwd)" LIB="$TOP/lib" YAML="$TOP/conf/graphs/rocksdb_plus.yaml" -preload_toplingdb \ No newline at end of file +preload_toplingdb diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_plus.yaml b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_plus.yaml index 989b8fefdf..f9a4bd8681 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_plus.yaml +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_plus.yaml @@ -151,4 +151,4 @@ DBOptions: max_manifest_file_size: 100M max_background_jobs: 8 compaction_readahead_size: 0 - memtable_as_log_index: true \ No newline at end of file + memtable_as_log_index: true diff --git a/pom.xml b/pom.xml index 1fa07660ee..08208f22e0 100644 --- a/pom.xml +++ b/pom.xml @@ -207,6 +207,9 @@ **/src/main/java/org/apache/hugegraph/pd/grpc/** **/src/main/java/org/apache/hugegraph/store/grpc/** + + **/library/*.html + **/library/*.css true From 6c82d7a55635bd8257d7a763f622ab0ccc0a6f77 Mon Sep 17 00:00:00 2001 From: gydeng Date: Tue, 2 Sep 2025 11:14:56 +0000 Subject: [PATCH 16/51] chore(server): adopt review suggestions from AI robots --- .../static/conf/graphs/rocksdb_plus.yaml | 4 +- .../src/assembly/travis/install-rocksdb.sh | 44 +++++++++++++++++++ .../store/rocksdb/RocksDBStdSessions.java | 6 ++- .../backend/store/rocksdb/RocksDBStore.java | 4 +- 4 files changed, 53 insertions(+), 5 deletions(-) diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_plus.yaml b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_plus.yaml index f9a4bd8681..d33d8da05a 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_plus.yaml +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_plus.yaml @@ -83,8 +83,8 @@ MemTableRepFactory: lookahead: 0 TableFactory: cspp_memtab_sst: - class: CSPPMemTabTable - params: # empty params + class: CSPPMemTabTable + params: # empty params bb: class: BlockBasedTable params: diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh index 53b6217609..b5ee210600 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh @@ -55,6 +55,48 @@ function extract_so_with_jar() { fi } +function extract_html_css_from_jar() { + local jar_file="$1" + local dest_dir="$2" + local abs_jar_path + local pipeline_status + local resource_target="/dev/shm/rocksdb_resource" + + if [ ! -f "$jar_file" ]; then + echo "Error: JAR file '$jar_file' does not exist." >&2 + return 1 + fi + + if ! mkdir -p "$dest_dir"; then + echo "Error: Cannot create destination directory '$dest_dir'." >&2 + return 1 + fi + + if [[ "$jar_file" == /* ]]; then + abs_jar_path="$jar_file" + else + abs_jar_path="$(pwd)/$jar_file" + fi + + unzip -j -o "$abs_jar_path" "*.html" "*.css" -d "$dest_dir" > /dev/null + pipeline_status=$? + + if [ $pipeline_status -eq 11 ]; then + echo "Notice: No .html or .css files found in '$jar_file'." >&2 + return 0 + elif [ $pipeline_status -ne 0 ]; then + echo "Error: unzip failed with exit code $pipeline_status" >&2 + return $pipeline_status + fi + + if ! mkdir -p "$resource_target"; then + echo "Error: Cannot create target directory '$resource_target'." >&2 + return 1 + fi + + cp -f "$dest_dir"/*.html "$dest_dir"/*.css "$resource_target" 2>/dev/null || true +} + function preload_toplingdb() { local jar_file=$(find $LIB -name "rocksdbjni*.jar") local dest_dir="$(pwd)"/$LIBRARY @@ -79,6 +121,8 @@ function preload_toplingdb() { extract_so_with_jar $jar_file $dest_dir export LD_LIBRARY_PATH=$dest_dir:$LD_LIBRARY_PATH export LD_PRELOAD=libjemalloc.so:librocksdbjni-linux64.so + + extract_html_css_from_jar $jar_file $dest_dir } VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java index c305d48819..8fbf3869e9 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java @@ -437,7 +437,8 @@ private static OpenedRocksDB openRocksDB(HugeConfig config, String dataPath, rocksdb = (RocksDB) openDBMethod.invoke(repo, converseOptionsToJsonString(dataPath, null)); - if (openHttp && BackendStoreProvider.GRAPH_STORE.equals(dbName)) { + if (Boolean.TRUE.equals(openHttp) && + BackendStoreProvider.GRAPH_STORE.equals(dbName)) { Method openHttpMethod = sidePluginRepoClass.getMethod("startHttpServer"); openHttpMethod.invoke(repo); @@ -542,7 +543,8 @@ private static OpenedRocksDB openRocksDB(HugeConfig config, converseOptionsToJsonString(dataPath, cfs), cfhs); - if (openHttp && BackendStoreProvider.GRAPH_STORE.equals(dbName)) { + if (Boolean.TRUE.equals(openHttp) && + BackendStoreProvider.GRAPH_STORE.equals(dbName)) { Method openHttpMethod = sidePluginRepoClass.getMethod("startHttpServer"); openHttpMethod.invoke(repo); diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStore.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStore.java index fb16044697..719b663b2c 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStore.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStore.java @@ -240,7 +240,9 @@ public synchronized void open(HugeConfig config) { openedDisks.add(disk); List tables = this.tableNames(e.getKey()); futures.add(openPool.submit(() -> { - this.open(config, disk, disk, tables, null, false); + this.open(config, disk, disk, tables, + config.get(RocksDBOptions.OPTION_PATH), + config.get(RocksDBOptions.OPEN_HTTP)); })); } } From 0151849b6dfd144d30b38be8721ff6096ee6314a Mon Sep 17 00:00:00 2001 From: gydeng Date: Tue, 9 Sep 2025 03:08:50 +0000 Subject: [PATCH 17/51] chore(server): add third-party package verification --- .../assembly/static/bin/preload-topling.sh | 25 ++- .../src/assembly/travis/install-rocksdb.sh | 25 ++- .../backend/store/rocksdb/RocksDBOptions.java | 2 +- .../store/rocksdb/RocksDBStdSessions.java | 57 ++++- .../unit/rocksdb/BaseRocksDBUnitTest.java | 2 +- .../unit/rocksdb/RocksDBSessionsTest.java | 2 +- install-dist/release-docs/LICENSE | 1 + .../LICENSE-rocksdbjni-8.10.2-SNAPSHOT.txt | 202 ++++++++++++++++++ .../scripts/dependency/known-dependencies.txt | 2 +- 9 files changed, 295 insertions(+), 23 deletions(-) create mode 100644 install-dist/release-docs/licenses/LICENSE-rocksdbjni-8.10.2-SNAPSHOT.txt diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh index c743ab91bd..396584984f 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh @@ -34,10 +34,12 @@ function extract_so_with_jar() { if [ ! -f "$jar_file" ]; then echo "'$jar_file' Not Exist" >&2 + return 1 fi if ! mkdir -p "$dest_dir"; then echo "Cannot mkdir '$dest_dir'" >&2 + return 1 fi if [[ "$jar_file" == /* ]]; then @@ -46,12 +48,16 @@ function extract_so_with_jar() { abs_jar_path="$(pwd)/$jar_file" fi - unzip -j -o "$abs_jar_path" "*.so" -d "$dest_dir" > /dev/null + unzip -j -o "$abs_jar_path" "*.so" -d "$dest_dir" > /dev/null 2>&1 pipeline_status=$? if [ $pipeline_status -ne 0 ]; then - # unzip provides specific exit codes that can be more descriptive. - echo "Error: Failed to extract .so files with unzip (Exit Code: $pipeline_status)" >&2 + if [ $pipeline_status -eq 11 ]; then + echo "Error: No .so files found in '$abs_jar_path' (unzip exit 11)" >&2 + else + echo "Error: unzip failed (exit $pipeline_status) for '$abs_jar_path'" >&2 + fi + return $pipeline_status fi } @@ -98,8 +104,17 @@ function extract_html_css_from_jar() { } function preload_toplingdb() { - local jar_file=$(find $LIB -name "rocksdbjni*.jar") - local dest_dir="$TOP/library" + local jar_file + local dest_dir + + # Find first rocksdbjni*.jar + jar_file=$(find "$LIB" -maxdepth 1 -type f -name "rocksdbjni*.jar" -print -quit || true) + if [ -z "${jar_file:-}" ]; then + echo "Error: No rocksdbjni*.jar found under '$LIB'" >&2 + return 1 + fi + + dest_dir="$TOP/library" # Check for Ubuntu 24.04+ and create a symlink for libaio if needed. # This is a workaround for software expecting the old libaio.so.1 name, diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh index b5ee210600..b9d6bbdad2 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh @@ -34,10 +34,12 @@ function extract_so_with_jar() { if [ ! -f "$jar_file" ]; then echo "'$jar_file' Not Exist" >&2 + return 1 fi if ! mkdir -p "$dest_dir"; then echo "Cannot mkdir '$dest_dir'" >&2 + return 1 fi if [[ "$jar_file" == /* ]]; then @@ -46,12 +48,16 @@ function extract_so_with_jar() { abs_jar_path="$(pwd)/$jar_file" fi - unzip -j -o "$abs_jar_path" "*.so" -d "$dest_dir" > /dev/null + unzip -j -o "$abs_jar_path" "*.so" -d "$dest_dir" > /dev/null 2>&1 pipeline_status=$? if [ $pipeline_status -ne 0 ]; then - # unzip provides specific exit codes that can be more descriptive. - echo "Error: Failed to extract .so files with unzip (Exit Code: $pipeline_status)" >&2 + if [ $pipeline_status -eq 11 ]; then + echo "Error: No .so files found in '$abs_jar_path' (unzip exit 11)" >&2 + else + echo "Error: unzip failed (exit $pipeline_status) for '$abs_jar_path'" >&2 + fi + return $pipeline_status fi } @@ -98,8 +104,17 @@ function extract_html_css_from_jar() { } function preload_toplingdb() { - local jar_file=$(find $LIB -name "rocksdbjni*.jar") - local dest_dir="$(pwd)"/$LIBRARY + local jar_file + local dest_dir + + # Find first rocksdbjni*.jar + jar_file=$(find "$LIB" -maxdepth 1 -type f -name "rocksdbjni*.jar" -print -quit || true) + if [ -z "${jar_file:-}" ]; then + echo "Error: No rocksdbjni*.jar found under '$LIB'" >&2 + return 1 + fi + + dest_dir="$(pwd)"/$LIBRARY # Check for Ubuntu 24.04+ and create a symlink for libaio if needed. # This is a workaround for software expecting the old libaio.so.1 name, diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java index 3819897285..6f54e2f439 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java @@ -89,7 +89,7 @@ public static synchronized RocksDBOptions instance() { public static final ConfigOption OPTION_PATH = new ConfigOption<>( "rocksdb.option_path", - "The path of the JSON file for configuring RocksDB options.", + "The YAML file for configuring Topling/RocksDB parameters", null, "" ); diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java index 8fbf3869e9..473690d80f 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java @@ -92,15 +92,34 @@ public class RocksDBStdSessions extends RocksDBSessions { private final AtomicInteger refCount; public RocksDBStdSessions(HugeConfig config, String database, String store, - String dataPath, String walPath, String optionPath, - Boolean openHttp) throws - RocksDBException { + String dataPath, String walPath, Object... optionalArgs) throws + RocksDBException { super(config, database, store); this.config = config; this.dataPath = dataPath; this.walPath = walPath; - this.optionPath = optionPath; - this.openHttp = openHttp; + + // Parse optional Args + if (optionalArgs.length >= 2) { + if (!(optionalArgs[0] instanceof String)) { + throw new IllegalArgumentException("Expected String for optionPath at position 0"); + } + if (!(optionalArgs[1] instanceof Boolean)) { + throw new IllegalArgumentException("Expected Boolean for openHttp at position 1"); + } + this.optionPath = (String) optionalArgs[0]; + this.openHttp = (Boolean) optionalArgs[1]; + } else if (optionalArgs.length == 1) { + if (!(optionalArgs[0] instanceof String)) { + throw new IllegalArgumentException("Expected String for optionPath at position 0"); + } + this.optionPath = (String) optionalArgs[0]; + this.openHttp = false; // 默认值 + } else { + this.optionPath = null; // 默认值 + this.openHttp = false; // 默认值 + } + this.rocksdb = RocksDBStdSessions.openRocksDB(config, dataPath, walPath, optionPath, openHttp); this.refCount = new AtomicInteger(1); @@ -108,14 +127,34 @@ public RocksDBStdSessions(HugeConfig config, String database, String store, public RocksDBStdSessions(HugeConfig config, String database, String store, String dataPath, String walPath, - List cfNames, String optionPath, Boolean openHttp) throws - RocksDBException { + List cfNames, Object... optionalArgs) throws + RocksDBException { super(config, database, store); this.config = config; this.dataPath = dataPath; this.walPath = walPath; - this.optionPath = optionPath; - this.openHttp = openHttp; + + // Parse optional Args + if (optionalArgs.length >= 2) { + if (!(optionalArgs[0] instanceof String)) { + throw new IllegalArgumentException("Expected String for optionPath at position 0"); + } + if (!(optionalArgs[1] instanceof Boolean)) { + throw new IllegalArgumentException("Expected Boolean for openHttp at position 1"); + } + this.optionPath = (String) optionalArgs[0]; + this.openHttp = (Boolean) optionalArgs[1]; + } else if (optionalArgs.length == 1) { + if (!(optionalArgs[0] instanceof String)) { + throw new IllegalArgumentException("Expected String for optionPath at position 0"); + } + this.optionPath = (String) optionalArgs[0]; + this.openHttp = false; + } else { + this.optionPath = null; + this.openHttp = false; + } + this.rocksdb = RocksDBStdSessions.openRocksDB(config, cfNames, dataPath, walPath, optionPath, openHttp); diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/BaseRocksDBUnitTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/BaseRocksDBUnitTest.java index 65bde3edf9..eb25fd8857 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/BaseRocksDBUnitTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/BaseRocksDBUnitTest.java @@ -109,7 +109,7 @@ protected static long getLong(byte[] bytes) { private static RocksDBSessions open(String table) throws RocksDBException { HugeConfig config = FakeObjects.newConfig(); RocksDBSessions rocks = new RocksDBStdSessions(config, "db", "store", - DB_PATH, DB_PATH, null, false); + DB_PATH, DB_PATH); rocks.createTable(table); return rocks; } diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/RocksDBSessionsTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/RocksDBSessionsTest.java index 34c226e46d..accc5e7f6b 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/RocksDBSessionsTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/RocksDBSessionsTest.java @@ -191,7 +191,7 @@ public void testIngestSst() throws RocksDBException { Assert.assertFalse(sstSessions.existsTable(TABLE2)); RocksDBSessions rocks = - new RocksDBStdSessions(config, "db", "store", sstPath, sstPath, null, false); + new RocksDBStdSessions(config, "db", "store", sstPath, sstPath); // Will ingest sst file of TABLE1 rocks.createTable(TABLE1); Assert.assertEquals(ImmutableList.of("1000"), diff --git a/install-dist/release-docs/LICENSE b/install-dist/release-docs/LICENSE index 5cbac87f44..b1a319339b 100644 --- a/install-dist/release-docs/LICENSE +++ b/install-dist/release-docs/LICENSE @@ -571,6 +571,7 @@ The text of each license is also included in licenses/LICENSE-[project].txt. https://central.sonatype.com/artifact/org.rocksdb/rocksdbjni/6.29.5 -> Apache 2.0 https://central.sonatype.com/artifact/org.rocksdb/rocksdbjni/7.2.2 -> Apache 2.0 https://central.sonatype.com/artifact/org.rocksdb/rocksdbjni/7.7.3 -> Apache 2.0 + https://github.com/hugegraph/toplingdb/packages/2550860 -> Apache 2.0 https://central.sonatype.com/artifact/org.fusesource/sigar/1.6.4 -> Apache 2.0 https://central.sonatype.com/artifact/io.prometheus/simpleclient/0.10.0 -> Apache 2.0 https://central.sonatype.com/artifact/io.prometheus/simpleclient_common/0.10.0 -> Apache 2.0 diff --git a/install-dist/release-docs/licenses/LICENSE-rocksdbjni-8.10.2-SNAPSHOT.txt b/install-dist/release-docs/licenses/LICENSE-rocksdbjni-8.10.2-SNAPSHOT.txt new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/install-dist/release-docs/licenses/LICENSE-rocksdbjni-8.10.2-SNAPSHOT.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/install-dist/scripts/dependency/known-dependencies.txt b/install-dist/scripts/dependency/known-dependencies.txt index 6acee82358..76cff6f45b 100644 --- a/install-dist/scripts/dependency/known-dependencies.txt +++ b/install-dist/scripts/dependency/known-dependencies.txt @@ -419,8 +419,8 @@ reporter-config3-3.0.3.jar reporter-config-base-3.0.3.jar rewriting-9.0-9.0.20190305.jar rocksdbjni-6.29.5.jar -rocksdbjni-7.2.2.jar rocksdbjni-7.7.3.jar +rocksdbjni-8.10.2-SNAPSHOT.jar scala-java8-compat_2.12-0.8.0.jar scala-library-2.12.7.jar scala-reflect-2.12.7.jar From d33aca33dcfa4ed0a738380cb945fd58552b19ec Mon Sep 17 00:00:00 2001 From: gydeng Date: Wed, 10 Sep 2025 09:11:43 +0000 Subject: [PATCH 18/51] chore(server): add unified error capture in the script to facilitate positioning --- .../src/assembly/static/bin/preload-topling.sh | 7 ++++++- .../hugegraph-dist/src/assembly/travis/install-rocksdb.sh | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh index 396584984f..54d0704e9d 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh @@ -16,6 +16,11 @@ # limitations under the License. # +set -Eeuo pipefail +IFS=$'\n\t' +# Unified error capture for easy positioning +trap 'echo "[preload-topling] error at line ${LINENO}: ${BASH_COMMAND}" >&2' ERR + function abs_path() { SOURCE="${BASH_SOURCE[0]}" while [[ -h "$SOURCE" ]]; do @@ -134,7 +139,7 @@ function preload_toplingdb() { fi extract_so_with_jar $jar_file $dest_dir - export LD_LIBRARY_PATH=$dest_dir:$LD_LIBRARY_PATH + export LD_LIBRARY_PATH="${dest_dir}:${LD_LIBRARY_PATH:-}" export LD_PRELOAD=libjemalloc.so:librocksdbjni-linux64.so extract_html_css_from_jar $jar_file $dest_dir diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh index b9d6bbdad2..3c655d2708 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh @@ -16,6 +16,11 @@ # limitations under the License. # +set -Eeuo pipefail +IFS=$'\n\t' +# Unified error capture for easy positioning +trap 'echo "[install-rocksdb] error at line ${LINENO}: ${BASH_COMMAND}" >&2' ERR + function abs_path() { SOURCE="${BASH_SOURCE[0]}" while [[ -h "$SOURCE" ]]; do @@ -134,7 +139,7 @@ function preload_toplingdb() { fi extract_so_with_jar $jar_file $dest_dir - export LD_LIBRARY_PATH=$dest_dir:$LD_LIBRARY_PATH + export LD_LIBRARY_PATH="${dest_dir}:${LD_LIBRARY_PATH:-}" export LD_PRELOAD=libjemalloc.so:librocksdbjni-linux64.so extract_html_css_from_jar $jar_file $dest_dir From d107e74e069054556a7ad8ebaa24de70352bc47b Mon Sep 17 00:00:00 2001 From: "pull[bot]" <39814207+pull[bot]@users.noreply.github.com> Date: Mon, 22 Sep 2025 12:22:02 +0800 Subject: [PATCH 19/51] chore: improve maven Reproducible Builds (#2874) (#42) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Hervé Boutemy --- .../hugegraph-clustertest-dist/pom.xml | 1 - hugegraph-pd/hg-pd-dist/pom.xml | 1 - hugegraph-pd/hg-pd-service/pom.xml | 2 +- hugegraph-server/hugegraph-core/pom.xml | 1 - hugegraph-server/hugegraph-dist/pom.xml | 9 +++++---- hugegraph-store/hg-store-cli/pom.xml | 2 +- hugegraph-store/hg-store-dist/pom.xml | 1 - hugegraph-store/hg-store-node/pom.xml | 2 +- pom.xml | 8 ++++++++ 9 files changed, 16 insertions(+), 11 deletions(-) diff --git a/hugegraph-cluster-test/hugegraph-clustertest-dist/pom.xml b/hugegraph-cluster-test/hugegraph-clustertest-dist/pom.xml index 20e3efc599..7ec79efba0 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-dist/pom.xml +++ b/hugegraph-cluster-test/hugegraph-clustertest-dist/pom.xml @@ -42,7 +42,6 @@ maven-assembly-plugin - 2.4 assembly-hugegraph-ct diff --git a/hugegraph-pd/hg-pd-dist/pom.xml b/hugegraph-pd/hg-pd-dist/pom.xml index 251ec8bcb7..c0e3b11d72 100644 --- a/hugegraph-pd/hg-pd-dist/pom.xml +++ b/hugegraph-pd/hg-pd-dist/pom.xml @@ -48,7 +48,6 @@ maven-assembly-plugin - 2.4 assembly-hugegraph-pd diff --git a/hugegraph-pd/hg-pd-service/pom.xml b/hugegraph-pd/hg-pd-service/pom.xml index 81b4568701..902a39d181 100644 --- a/hugegraph-pd/hg-pd-service/pom.xml +++ b/hugegraph-pd/hg-pd-service/pom.xml @@ -150,7 +150,7 @@ org.springframework.boot spring-boot-maven-plugin - 2.5.0 + 2.7.1 diff --git a/hugegraph-server/hugegraph-core/pom.xml b/hugegraph-server/hugegraph-core/pom.xml index 5ec2185ab0..83bd141ce2 100644 --- a/hugegraph-server/hugegraph-core/pom.xml +++ b/hugegraph-server/hugegraph-core/pom.xml @@ -359,7 +359,6 @@ org.apache.maven.plugins maven-jar-plugin - 3.0.2 true diff --git a/hugegraph-server/hugegraph-dist/pom.xml b/hugegraph-server/hugegraph-dist/pom.xml index 7f4dd454c9..324d253dc5 100644 --- a/hugegraph-server/hugegraph-dist/pom.xml +++ b/hugegraph-server/hugegraph-dist/pom.xml @@ -165,7 +165,6 @@ org.apache.maven.plugins maven-assembly-plugin - 2.4 assembly-hugegraph @@ -177,9 +176,11 @@ false false ${top.level.dir} - - ${assembly.descriptor.dir}/assembly.xml - + + + ${assembly.descriptor.dir}/assembly.xml + + ${final.name} diff --git a/hugegraph-store/hg-store-cli/pom.xml b/hugegraph-store/hg-store-cli/pom.xml index 84de815696..5ad84966f8 100644 --- a/hugegraph-store/hg-store-cli/pom.xml +++ b/hugegraph-store/hg-store-cli/pom.xml @@ -67,7 +67,7 @@ org.springframework.boot spring-boot-maven-plugin - 2.5.14 + 2.7.1 diff --git a/hugegraph-store/hg-store-dist/pom.xml b/hugegraph-store/hg-store-dist/pom.xml index b0988d8aab..12acb0ba9e 100644 --- a/hugegraph-store/hg-store-dist/pom.xml +++ b/hugegraph-store/hg-store-dist/pom.xml @@ -48,7 +48,6 @@ maven-assembly-plugin - 2.4 assembly-hugegraph-store diff --git a/hugegraph-store/hg-store-node/pom.xml b/hugegraph-store/hg-store-node/pom.xml index 5406b1c8a4..aff68d0db0 100644 --- a/hugegraph-store/hg-store-node/pom.xml +++ b/hugegraph-store/hg-store-node/pom.xml @@ -156,7 +156,7 @@ org.springframework.boot spring-boot-maven-plugin - 2.5.14 + 2.7.1 diff --git a/pom.xml b/pom.xml index 1fa07660ee..3b9d71ba05 100644 --- a/pom.xml +++ b/pom.xml @@ -251,6 +251,14 @@ + + maven-remote-resources-plugin + 3.3.0 + + + maven-shade-plugin + 3.6.1 + From 0c5eb2fd2f4b3d5eedfb013c42602e785cef8257 Mon Sep 17 00:00:00 2001 From: gydeng Date: Thu, 25 Sep 2025 12:04:29 +0000 Subject: [PATCH 20/51] chore(server): extract redundant code from scripts into common functions --- .../src/assembly/static/bin/common-topling.sh | 144 ++++++++++++++++++ .../assembly/static/bin/preload-topling.sh | 128 +--------------- .../src/assembly/travis/install-rocksdb.sh | 137 +---------------- .../store/rocksdb/RocksDBStdSessions.java | 38 +++-- 4 files changed, 179 insertions(+), 268 deletions(-) create mode 100644 hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh new file mode 100644 index 0000000000..e991cde30c --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh @@ -0,0 +1,144 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -Eeuo pipefail +IFS=$'\n\t' +trap 'echo "[common-topling] error at line ${LINENO}: ${BASH_COMMAND}" >&2' ERR + +function abs_path() { + local SOURCE="${BASH_SOURCE[0]}" + while [[ -h "$SOURCE" ]]; do + local DIR="$(cd -P "$(dirname "$SOURCE")" && pwd)" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + done + cd -P "$(dirname "$SOURCE")" && pwd +} + +function extract_so_with_jar() { + local jar_file="$1" + local dest_dir="$2" + local abs_jar_path + + if [ ! -f "$jar_file" ]; then + echo "'$jar_file' Not Exist" >&2 + return 1 + fi + + mkdir -p "$dest_dir" || { + echo "Cannot mkdir '$dest_dir'" >&2 + return 1 + } + + abs_jar_path="$(realpath "$jar_file")" + unzip -j -o "$abs_jar_path" "*.so" -d "$dest_dir" > /dev/null 2>&1 || { + local code=$? + if [ $code -eq 11 ]; then + echo "Error: No .so files found in '$abs_jar_path' (unzip exit 11)" >&2 + else + echo "Error: unzip failed (exit $code) for '$abs_jar_path'" >&2 + fi + return $code + } +} + +function extract_html_css_from_jar() { + local jar_file="$1" + local dest_dir="$2" + local abs_jar_path + local resource_target="/dev/shm/rocksdb_resource" + + if [ ! -f "$jar_file" ]; then + echo "Error: JAR file '$jar_file' does not exist." >&2 + return 1 + fi + + mkdir -p "$dest_dir" || { + echo "Error: Cannot create destination directory '$dest_dir'." >&2 + return 1 + } + + abs_jar_path="$(realpath "$jar_file")" + unzip -j -o "$abs_jar_path" "*.html" "*.css" -d "$dest_dir" > /dev/null || { + local code=$? + if [ $code -eq 11 ]; then + echo "Notice: No .html or .css files found in '$jar_file'." >&2 + return 0 + else + echo "Error: unzip failed with exit code $code" >&2 + return $code + fi + } + + mkdir -p "$resource_target" || { + echo "Error: Cannot create target directory '$resource_target'." >&2 + return 1 + } + + cp -f "$dest_dir"/*.html "$dest_dir"/*.css "$resource_target" 2>/dev/null || true +} + +function ensure_libaio_symlink() { + # Check for Ubuntu 24.04+ and create a symlink for libaio if needed. + # This is a workaround for software expecting the old libaio.so.1 name, + # as it was renamed to libaio.so.1t64 in the new release. + # https://askubuntu.com/questions/1512196/libaio1-on-noble/1516639#1516639 + if [ -f /etc/os-release ]; then + . /etc/os-release + if [ "${ID:-}" = "ubuntu" ] && command -v dpkg >/dev/null 2>&1 && dpkg --compare-versions "${VERSION_ID:-0}" "ge" "24.04"; then + local libaio_link_target="/usr/lib/x86_64-linux-gnu/libaio.so.1" + if [ ! -e $libaio_link_target ]; then + echo "Ubuntu ${VERSION_ID:-?} detected. Creating compatibility symlink for libaio." + if [ "$EUID" -eq 0 ]; then + ln -sf /usr/lib/x86_64-linux-gnu/libaio.so.1t64 "$libaio_link_target" || true + elif command -v sudo >/dev/null 2>&1; then + sudo ln -sf /usr/lib/x86_64-linux-gnu/libaio.so.1t64 "$libaio_link_target" || true + else + echo "Warn: sudo not available, skip creating $libaio_link_target" >&2 + fi + else + echo "libaio.so.1 found, skip creating compatibility symlink" >&2 + fi + fi + fi +} + +function preload_toplingdb() { + local lib_dir="$1" + local dest_dir="$2" + + local jar_file + jar_file=$(find "$lib_dir" -maxdepth 1 -type f -name "rocksdbjni*.jar" -print -quit || true) + if [ -z "${jar_file:-}" ]; then + echo "Error: No rocksdbjni*.jar found under '$lib_dir'" >&2 + return 1 + fi + + ensure_libaio_symlink + extract_so_with_jar "$jar_file" "$dest_dir" + export LD_LIBRARY_PATH="$dest_dir${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + if [ -f "$dest_dir/librocksdbjni-linux64.so" ]; then + export LD_PRELOAD="librocksdbjni-linux64.so${LD_PRELOAD:+:$LD_PRELOAD}" + fi + if command -v ldconfig >/dev/null 2>&1; then + jemalloc_found=$(ldconfig -p 2>/dev/null | grep -F 'libjemalloc.so' || true) + if [ -n "$jemalloc_found" ]; then + export LD_PRELOAD="libjemalloc.so${LD_PRELOAD:+:$LD_PRELOAD}" + fi + fi + extract_html_css_from_jar "$jar_file" "$dest_dir" +} diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh index 54d0704e9d..f7a5895fc8 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh @@ -21,133 +21,11 @@ IFS=$'\n\t' # Unified error capture for easy positioning trap 'echo "[preload-topling] error at line ${LINENO}: ${BASH_COMMAND}" >&2' ERR -function abs_path() { - SOURCE="${BASH_SOURCE[0]}" - while [[ -h "$SOURCE" ]]; do - DIR="$(cd -P "$(dirname "$SOURCE")" && pwd)" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" - done - cd -P "$(dirname "$SOURCE")" && pwd -} - -function extract_so_with_jar() { - local jar_file="$1" - local dest_dir="$2" - local abs_jar_path - local pipeline_status - - if [ ! -f "$jar_file" ]; then - echo "'$jar_file' Not Exist" >&2 - return 1 - fi - - if ! mkdir -p "$dest_dir"; then - echo "Cannot mkdir '$dest_dir'" >&2 - return 1 - fi - - if [[ "$jar_file" == /* ]]; then - abs_jar_path="$jar_file" - else - abs_jar_path="$(pwd)/$jar_file" - fi - - unzip -j -o "$abs_jar_path" "*.so" -d "$dest_dir" > /dev/null 2>&1 - pipeline_status=$? - - if [ $pipeline_status -ne 0 ]; then - if [ $pipeline_status -eq 11 ]; then - echo "Error: No .so files found in '$abs_jar_path' (unzip exit 11)" >&2 - else - echo "Error: unzip failed (exit $pipeline_status) for '$abs_jar_path'" >&2 - fi - return $pipeline_status - fi -} - -function extract_html_css_from_jar() { - local jar_file="$1" - local dest_dir="$2" - local abs_jar_path - local pipeline_status - local resource_target="/dev/shm/rocksdb_resource" - - if [ ! -f "$jar_file" ]; then - echo "Error: JAR file '$jar_file' does not exist." >&2 - return 1 - fi - - if ! mkdir -p "$dest_dir"; then - echo "Error: Cannot create destination directory '$dest_dir'." >&2 - return 1 - fi - - if [[ "$jar_file" == /* ]]; then - abs_jar_path="$jar_file" - else - abs_jar_path="$(pwd)/$jar_file" - fi - - unzip -j -o "$abs_jar_path" "*.html" "*.css" -d "$dest_dir" > /dev/null - pipeline_status=$? - - if [ $pipeline_status -eq 11 ]; then - echo "Notice: No .html or .css files found in '$jar_file'." >&2 - return 0 - elif [ $pipeline_status -ne 0 ]; then - echo "Error: unzip failed with exit code $pipeline_status" >&2 - return $pipeline_status - fi - - if ! mkdir -p "$resource_target"; then - echo "Error: Cannot create target directory '$resource_target'." >&2 - return 1 - fi - - cp -f "$dest_dir"/*.html "$dest_dir"/*.css "$resource_target" 2>/dev/null || true -} - -function preload_toplingdb() { - local jar_file - local dest_dir - - # Find first rocksdbjni*.jar - jar_file=$(find "$LIB" -maxdepth 1 -type f -name "rocksdbjni*.jar" -print -quit || true) - if [ -z "${jar_file:-}" ]; then - echo "Error: No rocksdbjni*.jar found under '$LIB'" >&2 - return 1 - fi - - dest_dir="$TOP/library" - - # Check for Ubuntu 24.04+ and create a symlink for libaio if needed. - # This is a workaround for software expecting the old libaio.so.1 name, - # as it was renamed to libaio.so.1t64 in the new release. - # https://askubuntu.com/questions/1512196/libaio1-on-noble/1516639#1516639 - if [ -f /etc/os-release ]; then - # Source the os-release file to get ID and VERSION_ID variables - . /etc/os-release - # Use dpkg to reliably compare version numbers - if [ "$ID" = "ubuntu" ] && dpkg --compare-versions "$VERSION_ID" "ge" "24.04"; then - local libaio_link_target="/usr/lib/x86_64-linux-gnu/libaio.so.1" - if [ ! -e "$libaio_link_target" ]; then - echo "Ubuntu 24.04 or newer detected. Creating compatibility symlink for libaio." - sudo ln -s /usr/lib/x86_64-linux-gnu/libaio.so.1t64 "$libaio_link_target" - fi - fi - fi - - extract_so_with_jar $jar_file $dest_dir - export LD_LIBRARY_PATH="${dest_dir}:${LD_LIBRARY_PATH:-}" - export LD_PRELOAD=libjemalloc.so:librocksdbjni-linux64.so - - extract_html_css_from_jar $jar_file $dest_dir -} +source "$(dirname "${BASH_SOURCE[0]}")/common-topling.sh" BIN=$(abs_path) TOP="$(cd "$BIN"/../ && pwd)" LIB="$TOP/lib" -YAML="$TOP/conf/graphs/rocksdb_plus.yaml" +DEST_DIR="$TOP/library" -preload_toplingdb +preload_toplingdb "$LIB" "$DEST_DIR" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh index 3c655d2708..8d5b19f292 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh @@ -21,135 +21,12 @@ IFS=$'\n\t' # Unified error capture for easy positioning trap 'echo "[install-rocksdb] error at line ${LINENO}: ${BASH_COMMAND}" >&2' ERR -function abs_path() { - SOURCE="${BASH_SOURCE[0]}" - while [[ -h "$SOURCE" ]]; do - DIR="$(cd -P "$(dirname "$SOURCE")" && pwd)" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" - done - cd -P "$(dirname "$SOURCE")" && pwd -} - -function extract_so_with_jar() { - local jar_file="$1" - local dest_dir="$2" - local abs_jar_path - local pipeline_status - - if [ ! -f "$jar_file" ]; then - echo "'$jar_file' Not Exist" >&2 - return 1 - fi - - if ! mkdir -p "$dest_dir"; then - echo "Cannot mkdir '$dest_dir'" >&2 - return 1 - fi - - if [[ "$jar_file" == /* ]]; then - abs_jar_path="$jar_file" - else - abs_jar_path="$(pwd)/$jar_file" - fi - - unzip -j -o "$abs_jar_path" "*.so" -d "$dest_dir" > /dev/null 2>&1 - pipeline_status=$? - - if [ $pipeline_status -ne 0 ]; then - if [ $pipeline_status -eq 11 ]; then - echo "Error: No .so files found in '$abs_jar_path' (unzip exit 11)" >&2 - else - echo "Error: unzip failed (exit $pipeline_status) for '$abs_jar_path'" >&2 - fi - return $pipeline_status - fi -} - -function extract_html_css_from_jar() { - local jar_file="$1" - local dest_dir="$2" - local abs_jar_path - local pipeline_status - local resource_target="/dev/shm/rocksdb_resource" - - if [ ! -f "$jar_file" ]; then - echo "Error: JAR file '$jar_file' does not exist." >&2 - return 1 - fi - - if ! mkdir -p "$dest_dir"; then - echo "Error: Cannot create destination directory '$dest_dir'." >&2 - return 1 - fi - - if [[ "$jar_file" == /* ]]; then - abs_jar_path="$jar_file" - else - abs_jar_path="$(pwd)/$jar_file" - fi - - unzip -j -o "$abs_jar_path" "*.html" "*.css" -d "$dest_dir" > /dev/null - pipeline_status=$? - - if [ $pipeline_status -eq 11 ]; then - echo "Notice: No .html or .css files found in '$jar_file'." >&2 - return 0 - elif [ $pipeline_status -ne 0 ]; then - echo "Error: unzip failed with exit code $pipeline_status" >&2 - return $pipeline_status - fi - - if ! mkdir -p "$resource_target"; then - echo "Error: Cannot create target directory '$resource_target'." >&2 - return 1 - fi - - cp -f "$dest_dir"/*.html "$dest_dir"/*.css "$resource_target" 2>/dev/null || true -} - -function preload_toplingdb() { - local jar_file - local dest_dir - - # Find first rocksdbjni*.jar - jar_file=$(find "$LIB" -maxdepth 1 -type f -name "rocksdbjni*.jar" -print -quit || true) - if [ -z "${jar_file:-}" ]; then - echo "Error: No rocksdbjni*.jar found under '$LIB'" >&2 - return 1 - fi - - dest_dir="$(pwd)"/$LIBRARY - - # Check for Ubuntu 24.04+ and create a symlink for libaio if needed. - # This is a workaround for software expecting the old libaio.so.1 name, - # as it was renamed to libaio.so.1t64 in the new release. - # https://askubuntu.com/questions/1512196/libaio1-on-noble/1516639#1516639 - if [ -f /etc/os-release ]; then - # Source the os-release file to get ID and VERSION_ID variables - . /etc/os-release - # Use dpkg to reliably compare version numbers - if [ "$ID" = "ubuntu" ] && dpkg --compare-versions "$VERSION_ID" "ge" "24.04"; then - local libaio_link_target="/usr/lib/x86_64-linux-gnu/libaio.so.1" - if [ ! -e "$libaio_link_target" ]; then - echo "Ubuntu 24.04 or newer detected. Creating compatibility symlink for libaio." - sudo ln -s /usr/lib/x86_64-linux-gnu/libaio.so.1t64 "$libaio_link_target" - fi - fi - fi - - extract_so_with_jar $jar_file $dest_dir - export LD_LIBRARY_PATH="${dest_dir}:${LD_LIBRARY_PATH:-}" - export LD_PRELOAD=libjemalloc.so:librocksdbjni-linux64.so - - extract_html_css_from_jar $jar_file $dest_dir -} - VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) -SERVER_DIR=hugegraph-server/apache-hugegraph-server-incubating-$VERSION -LIB=$SERVER_DIR/lib -DB_CONF=$SERVER_DIR/conf/graphs/db_bench_community.yaml -LIBRARY=$SERVER_DIR/library -GITHUB="https://github.com" +SERVER_DIR="$(pwd)/hugegraph-server/apache-hugegraph-server-incubating-$VERSION" +BIN="$SERVER_DIR/bin" +LIB="$SERVER_DIR/lib" +DEST_DIR="$SERVER_DIR/library" + +source "$BIN/common-topling.sh" -preload_toplingdb +preload_toplingdb "$LIB" "$DEST_DIR" diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java index 473690d80f..81bc01c437 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java @@ -114,10 +114,10 @@ public RocksDBStdSessions(HugeConfig config, String database, String store, throw new IllegalArgumentException("Expected String for optionPath at position 0"); } this.optionPath = (String) optionalArgs[0]; - this.openHttp = false; // 默认值 + this.openHttp = false; } else { - this.optionPath = null; // 默认值 - this.openHttp = false; // 默认值 + this.optionPath = null; + this.openHttp = false; } this.rocksdb = RocksDBStdSessions.openRocksDB(config, dataPath, walPath, optionPath, @@ -497,9 +497,16 @@ private static OpenedRocksDB openRocksDB(HugeConfig config, String dataPath, "Topling features (SidePluginRepo) are required but not found in the " + "rocksdbjni library.", e); - } catch (InvocationTargetException | InstantiationException | IllegalAccessException | - NoSuchMethodException e) { - throw new RuntimeException(e); + } catch (InvocationTargetException e) { + Throwable cause = e.getCause(); + if (cause instanceof RocksDBException) { + throw (RocksDBException) cause; + } else { + throw new RocksDBException( + "Failed to open DB with SidePluginRepo: " + cause.getMessage()); + } + } catch (InstantiationException | NoSuchMethodException | IllegalAccessException e) { + throw new RocksDBException("SidePluginRepo reflection error: " + e.getMessage()); } } else { rocksdb = RocksDB.open(options, dataPath); @@ -598,14 +605,16 @@ private static OpenedRocksDB openRocksDB(HugeConfig config, "Topling features (SidePluginRepo) are required but not found in the " + "rocksdbjni library.", e); - } catch (InvocationTargetException | InstantiationException | NoSuchMethodException | - IllegalAccessException e) { - Throwable rootCause = e; - - while (rootCause.getCause() != null && rootCause.getCause() != rootCause) { - rootCause = rootCause.getCause(); + } catch (InvocationTargetException e) { + Throwable cause = e.getCause(); + if (cause instanceof RocksDBException) { + throw (RocksDBException) cause; + } else { + throw new RocksDBException( + "Failed to open DB with SidePluginRepo: " + cause.getMessage()); } - throw new RocksDBException(rootCause.getMessage()); + } catch (InstantiationException | NoSuchMethodException | IllegalAccessException e) { + throw new RocksDBException("SidePluginRepo reflection error: " + e.getMessage()); } } else { // use rocksdb @@ -644,6 +653,9 @@ public static Set listCFs(String path) throws RocksDBException { } private static String converseOptionsToJsonString(String dataPath, List cfs) { + if (dataPath == null || dataPath.trim().isEmpty()) { + throw new IllegalArgumentException("dataPath cannot be null or empty"); + } // construct CFOptions JSONObject columnFamilies = new JSONObject(); // multi CFs From ed2d49d75783f7b1df071a2d5928cc14ba13f870 Mon Sep 17 00:00:00 2001 From: gydeng Date: Fri, 26 Sep 2025 02:35:36 +0000 Subject: [PATCH 21/51] chore(server): enhance script robustness --- .../src/assembly/static/bin/common-topling.sh | 29 ++++++++++++++----- .../assembly/static/bin/preload-topling.sh | 15 ++++++++-- .../src/assembly/travis/install-rocksdb.sh | 15 +++++++++- 3 files changed, 47 insertions(+), 12 deletions(-) diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh index e991cde30c..c4d77f410b 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh @@ -44,7 +44,11 @@ function extract_so_with_jar() { return 1 } - abs_jar_path="$(realpath "$jar_file")" + if command -v realpath >/dev/null 2>&1; then + abs_jar_path="$(realpath "$jar_file")" + else + abs_jar_path="$(readlink -f "$jar_file")" + fi unzip -j -o "$abs_jar_path" "*.so" -d "$dest_dir" > /dev/null 2>&1 || { local code=$? if [ $code -eq 11 ]; then @@ -72,7 +76,11 @@ function extract_html_css_from_jar() { return 1 } - abs_jar_path="$(realpath "$jar_file")" + if command -v realpath >/dev/null 2>&1; then + abs_jar_path="$(realpath "$jar_file")" + else + abs_jar_path="$(readlink -f "$jar_file")" + fi unzip -j -o "$abs_jar_path" "*.html" "*.css" -d "$dest_dir" > /dev/null || { local code=$? if [ $code -eq 11 ]; then @@ -101,14 +109,18 @@ function ensure_libaio_symlink() { . /etc/os-release if [ "${ID:-}" = "ubuntu" ] && command -v dpkg >/dev/null 2>&1 && dpkg --compare-versions "${VERSION_ID:-0}" "ge" "24.04"; then local libaio_link_target="/usr/lib/x86_64-linux-gnu/libaio.so.1" - if [ ! -e $libaio_link_target ]; then + if [ ! -e "$libaio_link_target" ]; then echo "Ubuntu ${VERSION_ID:-?} detected. Creating compatibility symlink for libaio." - if [ "$EUID" -eq 0 ]; then - ln -sf /usr/lib/x86_64-linux-gnu/libaio.so.1t64 "$libaio_link_target" || true - elif command -v sudo >/dev/null 2>&1; then - sudo ln -sf /usr/lib/x86_64-linux-gnu/libaio.so.1t64 "$libaio_link_target" || true + if [ -e /usr/lib/x86_64-linux-gnu/libaio.so.1t64 ]; then + if [ "$EUID" -eq 0 ]; then + ln -sf /usr/lib/x86_64-linux-gnu/libaio.so.1t64 "$libaio_link_target" || true + elif command -v sudo >/dev/null 2>&1; then + sudo ln -sf /usr/lib/x86_64-linux-gnu/libaio.so.1t64 "$libaio_link_target" || true + else + echo "Warn: sudo not available, skip creating $libaio_link_target" >&2 + fi else - echo "Warn: sudo not available, skip creating $libaio_link_target" >&2 + echo "Warn: libaio.so.1t64 not found, skip creating compat symlink" >&2 fi else echo "libaio.so.1 found, skip creating compatibility symlink" >&2 @@ -135,6 +147,7 @@ function preload_toplingdb() { export LD_PRELOAD="librocksdbjni-linux64.so${LD_PRELOAD:+:$LD_PRELOAD}" fi if command -v ldconfig >/dev/null 2>&1; then + local jemalloc_found jemalloc_found=$(ldconfig -p 2>/dev/null | grep -F 'libjemalloc.so' || true) if [ -n "$jemalloc_found" ]; then export LD_PRELOAD="libjemalloc.so${LD_PRELOAD:+:$LD_PRELOAD}" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh index f7a5895fc8..1caced578d 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh @@ -21,11 +21,20 @@ IFS=$'\n\t' # Unified error capture for easy positioning trap 'echo "[preload-topling] error at line ${LINENO}: ${BASH_COMMAND}" >&2' ERR -source "$(dirname "${BASH_SOURCE[0]}")/common-topling.sh" - -BIN=$(abs_path) +BIN="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TOP="$(cd "$BIN"/../ && pwd)" LIB="$TOP/lib" DEST_DIR="$TOP/library" +if [ ! -d "$LIB" ]; then + echo "Error: LIB dir not found: $LIB" >&2 + exit 1 +fi +if [ ! -f "$BIN/common-topling.sh" ]; then + echo "Error: common-topling.sh not found under: $BIN" >&2 + exit 1 +fi + +source "$BIN/common-topling.sh" +type preload_toplingdb >/dev/null 2>&1 || { echo "Error: function preload_toplingdb not found" >&2; exit 1; } preload_toplingdb "$LIB" "$DEST_DIR" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh index 8d5b19f292..753eb45acc 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh @@ -27,6 +27,19 @@ BIN="$SERVER_DIR/bin" LIB="$SERVER_DIR/lib" DEST_DIR="$SERVER_DIR/library" -source "$BIN/common-topling.sh" +if [ ! -d "$SERVER_DIR" ]; then + echo "Error: SERVER_DIR not found: $SERVER_DIR" >&2 + exit 1 +fi +if [ ! -d "$LIB" ]; then + echo "Error: LIB dir not found: $LIB" >&2 + exit 1 +fi +if [ ! -f "$BIN/common-topling.sh" ]; then + echo "Error: common-topling.sh not found under: $BIN" >&2 + exit 1 +fi +source "$BIN/common-topling.sh" +type preload_toplingdb >/dev/null 2>&1 || { echo "Error: function preload_toplingdb not found" >&2; exit 1; } preload_toplingdb "$LIB" "$DEST_DIR" From 577430bd07d49cf95883fa08a3bc0c03f78551cb Mon Sep 17 00:00:00 2001 From: gydeng Date: Fri, 26 Sep 2025 04:43:09 +0000 Subject: [PATCH 22/51] chore(server): corrected resource copy logic --- .../src/assembly/static/bin/common-topling.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh index c4d77f410b..a00f18401a 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh @@ -20,9 +20,11 @@ IFS=$'\n\t' trap 'echo "[common-topling] error at line ${LINENO}: ${BASH_COMMAND}" >&2' ERR function abs_path() { - local SOURCE="${BASH_SOURCE[0]}" + local SOURCE + SOURCE="${BASH_SOURCE[0]}" while [[ -h "$SOURCE" ]]; do - local DIR="$(cd -P "$(dirname "$SOURCE")" && pwd)" + local DIR + DIR="$(cd -P "$(dirname "$SOURCE")" && pwd)" SOURCE="$(readlink "$SOURCE")" [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" done @@ -97,7 +99,12 @@ function extract_html_css_from_jar() { return 1 } - cp -f "$dest_dir"/*.html "$dest_dir"/*.css "$resource_target" 2>/dev/null || true + if compgen -G "$dest_dir"/*.html >/dev/null 2>&1; then + cp -f "$dest_dir"/*.html "$resource_target"/ + fi + if compgen -G "$dest_dir"/*.css >/dev/null 2>&1; then + cp -f "$dest_dir"/*.css "$resource_target"/ + fi } function ensure_libaio_symlink() { From 4a591e7a63a5c4758e38e2a388f78a0aa8c616a6 Mon Sep 17 00:00:00 2001 From: gydeng Date: Fri, 26 Sep 2025 05:20:20 +0000 Subject: [PATCH 23/51] chore(server): avoid repeated expansion of LD_PRELOAD caused by multiple calls --- .../src/assembly/static/bin/common-topling.sh | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh index a00f18401a..7a1353c831 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh @@ -51,6 +51,10 @@ function extract_so_with_jar() { else abs_jar_path="$(readlink -f "$jar_file")" fi + if ! command -v unzip >/dev/null 2>&1; then + echo "Error: 'unzip' command not found. Please install unzip." >&2 + return 1 + fi unzip -j -o "$abs_jar_path" "*.so" -d "$dest_dir" > /dev/null 2>&1 || { local code=$? if [ $code -eq 11 ]; then @@ -83,6 +87,10 @@ function extract_html_css_from_jar() { else abs_jar_path="$(readlink -f "$jar_file")" fi + if ! command -v unzip >/dev/null 2>&1; then + echo "Error: 'unzip' command not found. Please install unzip." >&2 + return 1 + fi unzip -j -o "$abs_jar_path" "*.html" "*.css" -d "$dest_dir" > /dev/null || { local code=$? if [ $code -eq 11 ]; then @@ -141,7 +149,7 @@ function preload_toplingdb() { local dest_dir="$2" local jar_file - jar_file=$(find "$lib_dir" -maxdepth 1 -type f -name "rocksdbjni*.jar" -print -quit || true) + jar_file=$(ls -1 "$lib_dir"/rocksdbjni*.jar 2>/dev/null | sort -V | tail -n1 || true) if [ -z "${jar_file:-}" ]; then echo "Error: No rocksdbjni*.jar found under '$lib_dir'" >&2 return 1 @@ -149,14 +157,21 @@ function preload_toplingdb() { ensure_libaio_symlink extract_so_with_jar "$jar_file" "$dest_dir" - export LD_LIBRARY_PATH="$dest_dir${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" - if [ -f "$dest_dir/librocksdbjni-linux64.so" ]; then - export LD_PRELOAD="librocksdbjni-linux64.so${LD_PRELOAD:+:$LD_PRELOAD}" + if [ -d "$dest_dir" ]; then + if [[ ":${LD_LIBRARY_PATH:-}:" != *":$dest_dir:"* ]]; then + export LD_LIBRARY_PATH="$dest_dir${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + fi + + if [ -f "$dest_dir/librocksdbjni-linux64.so" ] && [[ ":${LD_PRELOAD:-}:" != *":librocksdbjni-linux64.so:"* ]]; then + export LD_PRELOAD="librocksdbjni-linux64.so${LD_PRELOAD:+:$LD_PRELOAD}" + fi + else + echo "Warn: LD paths skipped, directory '$dest_dir' does not exist." >&2 fi if command -v ldconfig >/dev/null 2>&1; then local jemalloc_found jemalloc_found=$(ldconfig -p 2>/dev/null | grep -F 'libjemalloc.so' || true) - if [ -n "$jemalloc_found" ]; then + if [ -n "$jemalloc_found" ] && [[ ":${LD_PRELOAD:-}:" != *":libjemalloc.so:"* ]]; then export LD_PRELOAD="libjemalloc.so${LD_PRELOAD:+:$LD_PRELOAD}" fi fi From f5a0d400b308751a9e6106b0ba62c09fb300ae4c Mon Sep 17 00:00:00 2001 From: gydeng Date: Mon, 29 Sep 2025 10:42:02 +0000 Subject: [PATCH 24/51] chore(docs): add rocksdb-plus documentation --- docs/rocksdb-plus.md | 155 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 docs/rocksdb-plus.md diff --git a/docs/rocksdb-plus.md b/docs/rocksdb-plus.md new file mode 100644 index 0000000000..dc85b777fb --- /dev/null +++ b/docs/rocksdb-plus.md @@ -0,0 +1,155 @@ +# RocksDB Plus (ToplingDB) Support and Configuration + +- **Status**: Implemented +- **Pull Request**: [#15](https://github.com/hugegraph/hugegraph/pull/15) + + + +## Background knowledge + +[ToplingDB](https://github.com/topling/toplingdb) is a high-performance, cloud-native key-value store built as a fork of RocksDB. + +ToplingDB extends RocksDB with several advanced features: + +- **Searchable Compression**: ToplingDB introduces compression algorithms that preserve searchability, enabling efficient queries directly on compressed data. +- **SidePlugin Architecture**: It supports dynamic configuration via YAML files through a plugin system, allowing runtime tuning without recompilation. +- **Built-in Observability**: A lightweight HTTP server exposes internal metrics and configuration states, making it easier to monitor and debug storage behavior. +- **Distributed Compaction**: Designed for cloud environments, ToplingDB supports distributed compaction strategies to reduce write amplification and improve throughput. +- **Compatibility**: Drop-in replacement for RocksDB in most use cases. + + + +## Motivation + +Introduce a new optional storage component in HugeGraph to support RocksDB Plus ([ToplingDB](https://github.com/topling/toplingdb)), a configurable and observable extension of the RocksDB storage engine. + +RocksDB Plus resolves key limitations in HugeGraph’s current `rocksdbjni` integration, which relies heavily on hard-coding parameters and lacks runtime configurability and observability. + +By enabling YAML-based configuration and exposing a Web Server interface, RocksDB Plus allows users to fine-tune performance and monitor engine behavior without modifying code or restarting services. + +This is especially valuable in environments where storage workloads vary across deployments, and where operational transparency is critical for debugging and optimization. + +For example, in production clusters with heterogeneous hardware or mixed graph workloads, users can dynamically adjust compaction, caching, and I/O settings to match their performance goals. + +Additionally, RocksDB Plus maintains full compatibility with the existing RocksDB API, allowing seamless migration and fallback. Users can opt into RocksDB Plus via configuration, without impacting legacy data or workflows. + +By supporting RocksDB Plus, HugeGraph empowers users with greater control over storage behavior, simplifies deployment through automated dynamic library loading, and enhances operational insight—all while preserving compatibility and ease of use. + + + +## Goals + +**Introduce RocksDB Plus as a configurable and observable alternative to RocksDB.** + +Enable users to select RocksDB Plus via configuration, allowing dynamic tuning through YAML files and real-time monitoring via Web Server—without sacrificing compatibility with existing RocksDB APIs. + + + +## Design + +### Configuration Parameters + +To support RocksDB Plus in HugeGraph, two new configuration parameters have been introduced: `rocksdb.option_path` and `rocksdb.open_http`. These options allow users to dynamically configure RocksDB behavior and enable real-time observability. + +#### `rocksdb.option_path`: External YAML Configuration + +This parameter allows users to specify a YAML file that defines RocksDB Plus settings such as compaction strategy, cache size, compression type, and more. + +- **Purpose**: Replace hard-coding parameters with flexible, file-based configuration. + +- **Usage**: Add the following line to your `hugegraph.properties` file: + + ``` + rocksdb.option_path=./conf/graphs/rocksdb_plus.yaml + ``` + + The specified YAML file will be automatically loaded during database initialization if RocksDB Plus is available. + + For details on the YAML structure and supported configuration fields, please refer to [SidePlugin](https://github.com/topling/sideplugin-wiki-en/wiki). + +- **Implementation**: During initialization, HugeGraph checks whether the configured JAR contains RocksDB Plus APIs. If so, it uses reflection to load the SidePluginRepo class and calls `importAutoFile(optionPath)` to parse the YAML file. The resulting configuration is applied to the RocksDB instance. + +- **Fallback**: If the YAML file is not provided or RocksDB Plus is unavailable, HugeGraph will fall back to standard RocksDB behavior. + + + +#### `rocksdb.open_http`: Enable Web Server for Observability + +This boolean flag controls whether the embedded Web Server in RocksDB Plus should be started. The server exposes runtime metrics, configuration status, and internal RocksDB statistics via a browser-accessible interface. + +- **Purpose**: Provide real-time visibility into the storage engine for debugging and performance tuning. + +- **Usage**: Add the following line to your `hugegraph.properties` file: + + ``` + rocksdb.open_http=true + ``` + + The listening port is defined in the YAML file specified by `option_path`, under the key `http.listening_ports`. + + To preview the Web Server interface and its layout, see [Web Server](https://github.com/topling/sideplugin-wiki-en/wiki/WebView). + +- **Implementation**: If `open_http` is set to true and the database instance is `GRAPH_STORE`, HugeGraph invokes `startHttpServer()` on the RocksDB Plus repo object. This exposes a browser-accessible dashboard for monitoring RocksDB internals. + +- **Scope**: For simplicity, the Web Server is only enabled for the `GRAPH_STORE` instance, which holds the main graph data. + + + +### Reflection-Based Loading Mechanism + +To support RocksDB Plus without introducing hard dependencies, HugeGraph uses Java reflection to detect and load enhanced APIs at runtime. + +During initialization, HugeGraph checks whether the current JAR contains the class `com.topling.sideplugin.SidePluginRepo`. If present, it assumes RocksDB Plus is available and proceeds to: + +1. **Load the SidePluginRepo class via reflection** This avoids compile-time coupling and allows fallback to standard RocksDB if the class is missing. +2. **Invoke** `importAutoFile(optionPath)` This method parses the YAML configuration file specified by `rocksdb.option_path`, dynamically applying storage engine parameters. +3. **Call** `open()` **with a JSON descriptor** The parsed configuration is converted to a JSON structure and passed to the RocksDB Plus engine to initialize the database. +4. **Optionally start the Web Server** If `rocksdb.open_http` is true and the instance is `GRAPH_STORE`, HugeGraph invokes `startHttpServer()` via reflection to enable observability. + +This design ensures that RocksDB Plus can be integrated as an optional enhancement, without breaking compatibility or requiring changes to the core HugeGraph codebase. + + + +## Impact + +### For Users + +User experience remains unchanged. +The RocksDB Plus integration is fully embedded into the existing startup scripts (`init-store.sh` and `start-hugegraph.sh`). Users only need to set `rocksdb.option_path` to specify the YAML file path and adjust its contents as needed to tune the storage engine. + + +### For Developers + +Developers need to make two adjustments to enable RocksDB Plus during development: + +1. **Maven Repository Configuration** Since RocksDB Plus is published via GitHub Packages, developers must add the GitHub repository to their `settings.xml` to fetch the correct JAR: + + ``` + + github + https://maven.pkg.github.com/hugegraph/toplingdb + + true + + + ``` + +2. **IDE Environment Setup** When using IDEs like IntelliJ IDEA, developers must configure runtime environment variables to preload the required native libraries. Developers can execute `preload-topling.sh` to extract the native libraries. The dynamic libraries and static resources required by the Web Server are extracted into the `library` directory located alongside the `bin` directory. + + In your IDE’s Run/Debug Configuration, set: + + ``` + LD_LIBRARY_PATH=/path/to/your/library:$LD_LIBRARY_PATH + LD_PRELOAD=libjemalloc.so:librocksdbjni.so + ``` + +These steps ensure that RocksDB Plus loads correctly in development environments and behaves consistently with production deployments. + + + +## Links + +* **ToplingDB**: https://github.com/topling/toplingdb +* **Configuration YAML of ToplingDB**: https://github.com/topling/sideplugin-wiki-en/wiki +* **Web Server of ToplingDB**: https://github.com/topling/sideplugin-wiki-en/wiki/WebView + From 2fd566ea82370df322b0705ecd9fbfaa3890d896 Mon Sep 17 00:00:00 2001 From: Tsukilc <153273766+Tsukilc@users.noreply.github.com> Date: Thu, 25 Sep 2025 16:59:34 +0800 Subject: [PATCH 25/51] docs: enhance docker instruction with auth opened graph (#2881) --- README.md | 34 ++++++++--------- .../hugegraph-dist/docker/README.md | 37 ++++++++++--------- 2 files changed, 34 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index f1b3923a74..dd50a85915 100644 --- a/README.md +++ b/README.md @@ -15,12 +15,11 @@ ## What is Apache HugeGraph? -[HugeGraph](https://hugegraph.apache.org/) is a fast and highly-scalable [graph database](https://en.wikipedia.org/wiki/Graph_database). -Billions of vertices and edges can be easily stored into and queried from HugeGraph due to its excellent OLTP capabilities. -HugeGraph is compliant with the [Apache TinkerPop 3](https://tinkerpop.apache.org/) framework allowing complicated graph queries to be +[HugeGraph](https://hugegraph.apache.org/) is a fast and highly-scalable [graph database](https://en.wikipedia.org/wiki/Graph_database). +Billions of vertices and edges can be easily stored into and queried from HugeGraph due to its excellent OLTP capabilities. +HugeGraph is compliant with the [Apache TinkerPop 3](https://tinkerpop.apache.org/) framework allowing complicated graph queries to be achieved through the powerful [Gremlin](https://tinkerpop.apache.org/gremlin.html) graph traversal language. - ## Features - Compliant to [Apache TinkerPop 3](https://tinkerpop.apache.org/), supports [Gremlin](https://tinkerpop.apache.org/gremlin.html) & [Cypher](https://en.wikipedia.org/wiki/Cypher) language @@ -30,7 +29,6 @@ achieved through the powerful [Gremlin](https://tinkerpop.apache.org/gremlin.htm - Integration with `Flink/Spark/HDFS`, and friendly to connect other big data platforms - Complete graph ecosystem (including both in/out-memory `Graph Computing` + `Graph Visualization & Tools` + `Graph Learning & AI`, see [here](#3-build-from-source)) - ## Quick Start ### 1. Docker (For Test) @@ -39,9 +37,9 @@ Use Docker to quickly start a HugeGraph server with `RocksDB` (in the background ``` # (Optional) -# 1. add "-e PASSWORD=xxx" to enable the auth system +# 1. remove "-e PASSWORD=xxx" to disable the auth system # 2. add "-e PRELOAD=true" to auto-load a sample graph -docker run -itd --name=graph -p 8080:8080 hugegraph/hugegraph:1.5.0 +docker run -itd --name=graph -e PASSWORD=xxx -p 8080:8080 hugegraph/hugegraph:1.5.0 ``` Please visit [doc page](https://hugegraph.apache.org/docs/quickstart/hugegraph-server/#3-deploy) or @@ -53,20 +51,21 @@ the [README](hugegraph-server/hugegraph-dist/docker/README.md) for more details. ### 2. Download -Visit [Download Page](https://hugegraph.apache.org/docs/download/download/) and refer the [doc](https://hugegraph.apache.org/docs/quickstart/hugegraph-server/#32-download-the-binary-tar-tarball) +Visit [Download Page](https://hugegraph.apache.org/docs/download/download/) and refer the [doc](https://hugegraph.apache.org/docs/quickstart/hugegraph-server/#32-download-the-binary-tar-tarball) to download the latest release package and start the server. -**Note:** if you want to use it in the production environment or expose it to the public network, must enable the [AuthSystem](https://hugegraph.apache.org/docs/config/config-authentication/) to ensure safe. +**Note:** if you want to use it in the production environment or expose it to the public network, must enable the [AuthSystem](https://hugegraph.apache.org/docs/config/config-authentication/) to ensure safe. ### 3. Build From Source -Visit [Build From Source Page](https://hugegraph.apache.org/docs/quickstart/hugegraph-server/#33-source-code-compilation) and follow the +Visit [Build From Source Page](https://hugegraph.apache.org/docs/quickstart/hugegraph-server/#33-source-code-compilation) and follow the steps to build the source code and start the server. The project [doc page](https://hugegraph.apache.org/docs/) contains more information on HugeGraph and provides detailed documentation for users. (Structure / Usage / API / Configs...) And here are links of other **HugeGraph** component/repositories: + 1. [hugegraph-toolchain](https://github.com/apache/hugegraph-toolchain) (graph tools **[loader](https://github.com/apache/hugegraph-toolchain/tree/master/hugegraph-loader)/[dashboard](https://github.com/apache/hugegraph-toolchain/tree/master/hugegraph-hubble)/[tool](https://github.com/apache/hugegraph-toolchain/tree/master/hugegraph-tools)/[client](https://github.com/apache/hugegraph-toolchain/tree/master/hugegraph-client)**) 2. [hugegraph-computer](https://github.com/apache/hugegraph-computer) (integrated **graph computing** system) 3. [hugegraph-ai](https://github.com/apache/incubator-hugegraph-ai) (integrated **Graph AI/LLM/KG** system) @@ -76,30 +75,27 @@ And here are links of other **HugeGraph** component/repositories: HugeGraph is licensed under [Apache 2.0 License](LICENSE). - ## Contributing -- Welcome to contribute to HugeGraph, please see [`How to Contribute`](CONTRIBUTING.md) & [Guidelines](https://hugegraph.apache.org/docs/contribution-guidelines/) for more information. -- Note: It's recommended to use [GitHub Desktop](https://desktop.github.com/) to greatly simplify the PR and commit process. +- Welcome to contribute to HugeGraph, please see [`How to Contribute`](CONTRIBUTING.md) & [Guidelines](https://hugegraph.apache.org/docs/contribution-guidelines/) for more information. +- Note: It's recommended to use [GitHub Desktop](https://desktop.github.com/) to greatly simplify the PR and commit process. - Thank you to all the people who already contributed to HugeGraph! [![contributors graph](https://contrib.rocks/image?repo=apache/hugegraph)](https://github.com/apache/incubator-hugegraph/graphs/contributors) - ## Thanks -HugeGraph relies on the [TinkerPop](http://tinkerpop.apache.org) framework, we refer to the storage structure of Titan and the schema definition of DataStax. +HugeGraph relies on the [TinkerPop](http://tinkerpop.apache.org) framework, we refer to the storage structure of Titan and the schema definition of DataStax. Thanks to TinkerPop, thanks to Titan, thanks to DataStax. Thanks to all other organizations or authors who contributed to the project. -You are welcome to contribute to HugeGraph, +You are welcome to contribute to HugeGraph, and we are looking forward to working with you to build an excellent open-source community. - ## Contact Us - [GitHub Issues](https://github.com/apache/hugegraph/issues): Feedback on usage issues and functional requirements (quick response) - - Feedback Email: [dev@hugegraph.apache.org](mailto:dev@hugegraph.apache.org) ([subscriber](https://hugegraph.apache.org/docs/contribution-guidelines/subscribe/) only) - - WeChat public account: Apache HugeGraph, welcome to scan this QR code to follow us. +- Feedback Email: [dev@hugegraph.apache.org](mailto:dev@hugegraph.apache.org) ([subscriber](https://hugegraph.apache.org/docs/contribution-guidelines/subscribe/) only) +- WeChat public account: Apache HugeGraph, welcome to scan this QR code to follow us. QR png diff --git a/hugegraph-server/hugegraph-dist/docker/README.md b/hugegraph-server/hugegraph-dist/docker/README.md index 5fac10de5c..6d1b6ad89b 100644 --- a/hugegraph-server/hugegraph-dist/docker/README.md +++ b/hugegraph-server/hugegraph-dist/docker/README.md @@ -1,9 +1,9 @@ # Deploy Hugegraph server with docker > Note: -> +> > 1. The docker image of hugegraph is a convenience release, not official distribution artifacts from ASF. You can find more details from [ASF Release Distribution Policy](https://infra.apache.org/release-distribution.html#dockerhub). -> +> > 2. Recommend to use `release tag` (like `1.3.0`/`1.5.0`) for the stable version. Use `latest` tag to experience the newest functions in development. ## 1. Deploy @@ -12,11 +12,11 @@ We can use docker to quickly start an inner HugeGraph server with RocksDB in the 1. Using docker run - Use `docker run -itd --name=graph -p 8080:8080 hugegraph/hugegraph:1.3.0` to start hugegraph server. + Use `docker run -itd --name=graph -p 8080:8080 hugegraph/hugegraph:1.3.0` to start hugegraph server. 2. Using docker compose - Certainly we can only deploy server without other instance. Additionally, if we want to manage other HugeGraph-related instances with `server` in a single file, we can deploy HugeGraph-related instances via `docker-compose up -d`. The `docker-compose.yaml` is as below: + Certainly we can only deploy server without other instance. Additionally, if we want to manage other HugeGraph-related instances with `server` in a single file, we can deploy HugeGraph-related instances via `docker-compose up -d`. The `docker-compose.yaml` is as below: ```yaml version: '3' @@ -35,12 +35,12 @@ If you want to customize the preloaded data, please mount the groovy scripts (no 1. Using docker run - Use `docker run -itd --name=graph -p 8080:8080 -e PRELOAD=true -v /path/to/script:/hugegraph-server/scripts/example.groovy hugegraph/hugegraph:1.3.0` - to start hugegraph server. + Use `docker run -itd --name=graph -p 8080:8080 -e PRELOAD=true -v /path/to/script:/hugegraph-server/scripts/example.groovy hugegraph/hugegraph:1.3.0` + to start hugegraph server. -2. Using docker compose +2. Using docker compose - We can also use `docker-compose up -d` to quickly start. The `docker-compose.yaml` is below. [example.groovy](https://github.com/apache/incubator-hugegraph/blob/master/hugegraph-server/hugegraph-dist/src/assembly/static/scripts/example.groovy) is a pre-defined script. If needed, we can mount a new `example.groovy` to preload different data: + We can also use `docker-compose up -d` to quickly start. The `docker-compose.yaml` is below. [example.groovy](https://github.com/apache/incubator-hugegraph/blob/master/hugegraph-server/hugegraph-dist/src/assembly/static/scripts/example.groovy) is a pre-defined script. If needed, we can mount a new `example.groovy` to preload different data: ```yaml version: '3' @@ -57,17 +57,17 @@ If you want to customize the preloaded data, please mount the groovy scripts (no 3. Using start-hugegraph.sh - If you deploy HugeGraph server without docker, you can also pass arguments using `-p`, like this: `bin/start-hugegraph.sh -p true`. + If you deploy HugeGraph server without docker, you can also pass arguments using `-p`, like this: `bin/start-hugegraph.sh -p true`. ## 3. Enable Authentication 1. Using docker run - Use `docker run -itd --name=graph -p 8080:8080 -e AUTH=true -e PASSWORD=123456 hugegraph/hugegraph:1.3.0` to enable the authentication and set the password with `-e AUTH=true -e PASSWORD=123456`. + Use `docker run -itd --name=graph -p 8080:8080 -e AUTH=true -e PASSWORD=xxx hugegraph/hugegraph:1.3.0` to enable the authentication and set the password with `-e AUTH=true -e PASSWORD=xxx`. 2. Using docker compose - Similarly, we can set the environment variables in the docker-compose.yaml: + Similarly, we can set the environment variables in the docker-compose.yaml: ```yaml version: '3' @@ -79,37 +79,38 @@ If you want to customize the preloaded data, please mount the groovy scripts (no - 8080:8080 environment: - AUTH=true - - PASSWORD=123456 + - PASSWORD=xxx ``` ## 4. Running Open-Telemetry-Collector > CAUTION: -> +> > The `docker-compose-trace.yaml` utilizes `Grafana` and `Grafana-Tempo`, both of them are licensed under [AGPL-3.0](https://www.gnu.org/licenses/agpl-3.0.en.html), you should be aware of and use them with caution. Currently, we mainly provide this template for everyone to **test** > + 1. Start Open-Telemetry-Collector ```bash cd hugegraph-server/hugegraph-dist/docker/example docker-compose -f docker-compose-trace.yaml -p hugegraph-trace up -d ``` - + 2. Active Open-Telemetry-Agent ```bash ./start-hugegraph.sh -y true ``` - + 3. Stop Open-Telemetry-Collector ```bash cd hugegraph-server/hugegraph-dist/docker/example docker-compose -f docker-compose-trace.yaml -p hugegraph-trace stop ``` - + 4. References - - [What is OpenTelemetry](https://opentelemetry.io/docs/what-is-opentelemetry/) + - [What is OpenTelemetry](https://opentelemetry.io/docs/what-is-opentelemetry/) - - [Tempo in Grafana](https://grafana.com/docs/tempo/latest/getting-started/tempo-in-grafana/) + - [Tempo in Grafana](https://grafana.com/docs/tempo/latest/getting-started/tempo-in-grafana/) From 0e564b1e8daeeb939706d2768831b09eacdb8474 Mon Sep 17 00:00:00 2001 From: Tsukilc <153273766+Tsukilc@users.noreply.github.com> Date: Sun, 28 Sep 2025 12:24:20 +0800 Subject: [PATCH 26/51] refactor: remove the package existing in java8 (#2792) * refactor: Delete the package existing in java8 * chore(format): remove custom line breaks --- .../hugegraph/auth/HugeFactoryAuthProxy.java | 9 ++-- .../security/HugeSecurityManager.java | 1 - .../org/apache/hugegraph/util/Reflection.java | 46 +++++++------------ 3 files changed, 20 insertions(+), 36 deletions(-) diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/auth/HugeFactoryAuthProxy.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/auth/HugeFactoryAuthProxy.java index 3ffaefb04f..41e9186d7b 100644 --- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/auth/HugeFactoryAuthProxy.java +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/auth/HugeFactoryAuthProxy.java @@ -66,10 +66,9 @@ public final class HugeFactoryAuthProxy { - private static final Logger LOG = Log.logger(HugeFactoryAuthProxy.class); public static final String GRAPH_FACTORY = "gremlin.graph=org.apache.hugegraph.auth.HugeFactoryAuthProxy"; - + private static final Logger LOG = Log.logger(HugeFactoryAuthProxy.class); private static final Set PROTECT_METHODS = ImmutableSet.of("instance"); private static final Map GRAPHS = new HashMap<>(); @@ -498,8 +497,6 @@ public static void filterCriticalSystemClasses() { Reflection.registerMethodsToFilter(loadClass("java.lang.ProcessImpl"), "forkAndExec", "setAccessible", "start"); - optionalMethodsToFilter("sun.invoke.util.BytecodeDescriptor", "parseMethod", "parseSig"); - optionalMethodsToFilter("sun.reflect.misc.MethodUtil", "invoke"); optionalMethodsToFilter("jdk.internal.reflect.MethodAccessor", "invoke"); optionalMethodsToFilter("jdk.internal.reflect.NativeMethodAccessorImpl", "invoke"); } @@ -636,8 +633,8 @@ public static void optionalMethodsToFilter(String className, String... methodNam try { clazz = Class.forName(className); } catch (ClassNotFoundException e) { - // TODO: we just ignore the exception, change it after we drop Java8 support - LOG.warn("Skip register class {} to filter", className); + LOG.debug("Internal class {} not found in this JDK implementation, skipping filter " + + "registration", className, e); } if (clazz != null) { Reflection.registerMethodsToFilter(clazz, methodNames); diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/security/HugeSecurityManager.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/security/HugeSecurityManager.java index f32491ece5..13ca801220 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/security/HugeSecurityManager.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/security/HugeSecurityManager.java @@ -48,7 +48,6 @@ public class HugeSecurityManager extends SecurityManager { private static final Set ACCEPT_CLASS_LOADERS = ImmutableSet.of( "groovy.lang.GroovyClassLoader", - "sun.reflect.DelegatingClassLoader", "jdk.internal.reflect.DelegatingClassLoader", "org.codehaus.groovy.reflection.SunClassLoader", "org.codehaus.groovy.runtime.callsite.CallSiteClassLoader", diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/util/Reflection.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/util/Reflection.java index f853824029..6e5fd07527 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/util/Reflection.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/util/Reflection.java @@ -33,49 +33,36 @@ public class Reflection { private static final Method REGISTER_FILEDS_TO_FILTER_METHOD; private static final Method REGISTER_METHODS_TO_FILTER_METHOD; - public static final String JDK_INTERNAL_REFLECT_REFLECTION = "jdk.internal.reflect.Reflection"; - public static final String SUN_REFLECT_REFLECTION = "sun.reflect.Reflection"; - static { Method registerFieldsToFilterMethodTemp = null; Method registerMethodsToFilterMethodTemp = null; Class reflectionClazzTemp = null; try { - reflectionClazzTemp = Class.forName(JDK_INTERNAL_REFLECT_REFLECTION); + reflectionClazzTemp = Class.forName("jdk.internal.reflect.Reflection"); + + registerFieldsToFilterMethodTemp = + reflectionClazzTemp.getMethod("registerFieldsToFilter", + Class.class, String[].class); + + registerMethodsToFilterMethodTemp = + reflectionClazzTemp.getMethod("registerMethodsToFilter", + Class.class, String[].class); } catch (ClassNotFoundException e) { - try { - reflectionClazzTemp = Class.forName(SUN_REFLECT_REFLECTION); - } catch (ClassNotFoundException ex) { - LOG.error("Can't find Reflection class", ex); - } + LOG.error("Can't find jdk.internal.reflect.Reflection class, " + + "please ensure you are using Java 11", e); + } catch (NoSuchMethodException e) { + LOG.error("Can't find reflection filter methods", e); } REFLECTION_CLAZZ = reflectionClazzTemp; - - if (REFLECTION_CLAZZ != null) { - try { - registerFieldsToFilterMethodTemp = - REFLECTION_CLAZZ.getMethod("registerFieldsToFilter", - Class.class, String[].class); - } catch (Throwable e) { - LOG.error("Can't find registerFieldsToFilter method", e); - } - - try { - registerMethodsToFilterMethodTemp = - REFLECTION_CLAZZ.getMethod("registerMethodsToFilter", - Class.class, String[].class); - } catch (NoSuchMethodException e) { - LOG.error("Can't find registerMethodsToFilter method", e); - } - } REGISTER_FILEDS_TO_FILTER_METHOD = registerFieldsToFilterMethodTemp; REGISTER_METHODS_TO_FILTER_METHOD = registerMethodsToFilterMethodTemp; } public static void registerFieldsToFilter(Class containingClass, String... fieldNames) { if (REGISTER_FILEDS_TO_FILTER_METHOD == null) { - throw new NotSupportException("Reflection.registerFieldsToFilter()"); + throw new NotSupportException("Reflection.registerFieldsToFilter() - " + + "requires Java 11 or higher"); } try { @@ -89,7 +76,8 @@ public static void registerFieldsToFilter(Class containingClass, String... fi public static void registerMethodsToFilter(Class containingClass, String... methodNames) { if (REGISTER_METHODS_TO_FILTER_METHOD == null) { - throw new NotSupportException("Reflection.registerMethodsToFilterMethod()"); + throw new NotSupportException("Reflection.registerMethodsToFilter() - " + + "requires Java 11 or higher"); } try { From 6fa2342dc3a8bddaf6c39502914f1c011b172080 Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 30 Sep 2025 17:11:36 +0800 Subject: [PATCH 27/51] docs: revise Docker usage instructions in README (#2882) Updated Docker instructions for HugeGraph server. --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index dd50a85915..168a6eaadd 100644 --- a/README.md +++ b/README.md @@ -37,8 +37,7 @@ Use Docker to quickly start a HugeGraph server with `RocksDB` (in the background ``` # (Optional) -# 1. remove "-e PASSWORD=xxx" to disable the auth system -# 2. add "-e PRELOAD=true" to auto-load a sample graph +# - add "-e PRELOAD=true" to auto-load a sample graph docker run -itd --name=graph -e PASSWORD=xxx -p 8080:8080 hugegraph/hugegraph:1.5.0 ``` From 4abed8ee113f0ce2827fde498dc25df64035cd9f Mon Sep 17 00:00:00 2001 From: gydeng Date: Wed, 8 Oct 2025 07:11:46 +0000 Subject: [PATCH 28/51] chore(docs): update rocksdb-plus documentation --- docs/rocksdb-plus.md | 90 ++++++++++++++++++++++++++++++-------------- 1 file changed, 62 insertions(+), 28 deletions(-) diff --git a/docs/rocksdb-plus.md b/docs/rocksdb-plus.md index dc85b777fb..93045a8f03 100644 --- a/docs/rocksdb-plus.md +++ b/docs/rocksdb-plus.md @@ -12,7 +12,7 @@ ToplingDB extends RocksDB with several advanced features: - **Searchable Compression**: ToplingDB introduces compression algorithms that preserve searchability, enabling efficient queries directly on compressed data. -- **SidePlugin Architecture**: It supports dynamic configuration via YAML files through a plugin system, allowing runtime tuning without recompilation. +- **SidePlugin Architecture**: It supports configuration via YAML files through a plugin system, allowing tuning parameters without recompilation. - **Built-in Observability**: A lightweight HTTP server exposes internal metrics and configuration states, making it easier to monitor and debug storage behavior. - **Distributed Compaction**: Designed for cloud environments, ToplingDB supports distributed compaction strategies to reduce write amplification and improve throughput. - **Compatibility**: Drop-in replacement for RocksDB in most use cases. @@ -29,7 +29,7 @@ By enabling YAML-based configuration and exposing a Web Server interface, RocksD This is especially valuable in environments where storage workloads vary across deployments, and where operational transparency is critical for debugging and optimization. -For example, in production clusters with heterogeneous hardware or mixed graph workloads, users can dynamically adjust compaction, caching, and I/O settings to match their performance goals. +For example, in production clusters with heterogeneous hardware or mixed graph workloads, users can adjust compaction, caching, and I/O settings to match their performance goals. Additionally, RocksDB Plus maintains full compatibility with the existing RocksDB API, allowing seamless migration and fallback. Users can opt into RocksDB Plus via configuration, without impacting legacy data or workflows. @@ -41,15 +41,14 @@ By supporting RocksDB Plus, HugeGraph empowers users with greater control over s **Introduce RocksDB Plus as a configurable and observable alternative to RocksDB.** -Enable users to select RocksDB Plus via configuration, allowing dynamic tuning through YAML files and real-time monitoring via Web Server—without sacrificing compatibility with existing RocksDB APIs. - +Enable users to select RocksDB Plus via configuration, allowing tuning parameters through YAML files without recompilation and real-time monitoring via Web Server—without sacrificing compatibility with existing RocksDB APIs. ## Design ### Configuration Parameters -To support RocksDB Plus in HugeGraph, two new configuration parameters have been introduced: `rocksdb.option_path` and `rocksdb.open_http`. These options allow users to dynamically configure RocksDB behavior and enable real-time observability. +To support RocksDB Plus in HugeGraph, two new configuration parameters have been introduced: `rocksdb.option_path` and `rocksdb.open_http`. These options allow users to configure RocksDB parameters and enable real-time observability. #### `rocksdb.option_path`: External YAML Configuration @@ -59,7 +58,7 @@ This parameter allows users to specify a YAML file that defines RocksDB Plus set - **Usage**: Add the following line to your `hugegraph.properties` file: - ``` + ```properties rocksdb.option_path=./conf/graphs/rocksdb_plus.yaml ``` @@ -81,11 +80,16 @@ This boolean flag controls whether the embedded Web Server in RocksDB Plus shoul - **Usage**: Add the following line to your `hugegraph.properties` file: - ``` + ```properties rocksdb.open_http=true ``` - The listening port is defined in the YAML file specified by `option_path`, under the key `http.listening_ports`. + The listening port is defined in the YAML file specified by `option_path`, under the key `http.listening_ports`: + ```yaml + http: + document_root: /dev/shm/rocksdb_resource + listening_ports: '2011' + ``` To preview the Web Server interface and its layout, see [Web Server](https://github.com/topling/sideplugin-wiki-en/wiki/WebView). @@ -93,6 +97,7 @@ This boolean flag controls whether the embedded Web Server in RocksDB Plus shoul - **Scope**: For simplicity, the Web Server is only enabled for the `GRAPH_STORE` instance, which holds the main graph data. +- **Security**: The Web Server does **not** provide built-in authentication. In production environments, configure firewalls or network access controls carefully to prevent unauthorized access. ### Reflection-Based Loading Mechanism @@ -102,9 +107,12 @@ To support RocksDB Plus without introducing hard dependencies, HugeGraph uses Ja During initialization, HugeGraph checks whether the current JAR contains the class `com.topling.sideplugin.SidePluginRepo`. If present, it assumes RocksDB Plus is available and proceeds to: 1. **Load the SidePluginRepo class via reflection** This avoids compile-time coupling and allows fallback to standard RocksDB if the class is missing. -2. **Invoke** `importAutoFile(optionPath)` This method parses the YAML configuration file specified by `rocksdb.option_path`, dynamically applying storage engine parameters. + * If the RocksDB Plus API cannot be found, HugeGraph silently falls back to the standard RocksDB API for startup. +2. **Invoke** `importAutoFile(optionPath)` This method parses the YAML configuration file specified by `rocksdb.option_path` to configure storage engine parameters. + * If the `option_path` is incorrect or parsing fails, RocksDB Plus throws an error and terminates the startup process. 3. **Call** `open()` **with a JSON descriptor** The parsed configuration is converted to a JSON structure and passed to the RocksDB Plus engine to initialize the database. 4. **Optionally start the Web Server** If `rocksdb.open_http` is true and the instance is `GRAPH_STORE`, HugeGraph invokes `startHttpServer()` via reflection to enable observability. + * If the Web Server cannot be started due to misconfiguration **or if the specified HTTP port is already in use**, RocksDB Plus throws an error and the startup process is terminated This design ensures that RocksDB Plus can be integrated as an optional enhancement, without breaking compatibility or requiring changes to the core HugeGraph codebase. @@ -114,7 +122,7 @@ This design ensures that RocksDB Plus can be integrated as an optional enhanceme ### For Users -User experience remains unchanged. +The way users operate remains unchanged by default, and adding RocksDB Plus configuration provides additional functionality. The RocksDB Plus integration is fully embedded into the existing startup scripts (`init-store.sh` and `start-hugegraph.sh`). Users only need to set `rocksdb.option_path` to specify the YAML file path and adjust its contents as needed to tune the storage engine. @@ -122,25 +130,51 @@ The RocksDB Plus integration is fully embedded into the existing startup scripts Developers need to make two adjustments to enable RocksDB Plus during development: -1. **Maven Repository Configuration** Since RocksDB Plus is published via GitHub Packages, developers must add the GitHub repository to their `settings.xml` to fetch the correct JAR: - - ``` - - github - https://maven.pkg.github.com/hugegraph/toplingdb - - true - - +1. **Maven Repository Configuration**: since RocksDB Plus is published via GitHub Packages, developers must add the GitHub repository to their `settings.xml` to fetch the correct JAR: + + ```xml + + + + + github + YOUR_GITHUB_ACTOR + YOUR_GITHUB_TOKEN + + + + + + ... + + ... + + + github + https://maven.pkg.github.com/hugegraph/toplingdb + + true + + + + + ``` -2. **IDE Environment Setup** When using IDEs like IntelliJ IDEA, developers must configure runtime environment variables to preload the required native libraries. Developers can execute `preload-topling.sh` to extract the native libraries. The dynamic libraries and static resources required by the Web Server are extracted into the `library` directory located alongside the `bin` directory. - +2. **IDE Environment Setup**: developers must configure runtime environment variables to preload required native libraries. + The `preload-topling.sh` script not only extracts the necessary dynamic libraries and web server static resources into the `library` directory next to the `bin` directory, + but also sets the required environment variables in the current process. + When executed in a terminal using `source preload-topling.sh`, these variables take effect immediately in that shell session. + + However, when launching HugeGraph from an IDE, the program typically runs in a separate process, + so environment variables defined in scripts run from the terminal are not inherited. + In this case, developers need to manually configure the IDE's run/debug environment variables to ensure proper preloading of native libraries. + In your IDE’s Run/Debug Configuration, set: - ``` - LD_LIBRARY_PATH=/path/to/your/library:$LD_LIBRARY_PATH - LD_PRELOAD=libjemalloc.so:librocksdbjni.so + ```bash + LD_LIBRARY_PATH="/path/to/your/library:${LD_LIBRARY_PATH}" + LD_PRELOAD="libjemalloc.so:librocksdbjni.so" ``` These steps ensure that RocksDB Plus loads correctly in development environments and behaves consistently with production deployments. @@ -149,7 +183,7 @@ These steps ensure that RocksDB Plus loads correctly in development environments ## Links -* **ToplingDB**: https://github.com/topling/toplingdb -* **Configuration YAML of ToplingDB**: https://github.com/topling/sideplugin-wiki-en/wiki -* **Web Server of ToplingDB**: https://github.com/topling/sideplugin-wiki-en/wiki/WebView +* **ToplingDB**: [https://github.com/topling/toplingdb](https://github.com/topling/toplingdb) +* **Configuration YAML of ToplingDB**: [https://github.com/topling/sideplugin-wiki-en/wiki](https://github.com/topling/sideplugin-wiki-en/wiki) +* **Web Server of ToplingDB**: [https://github.com/topling/sideplugin-wiki-en/wiki/WebView](https://github.com/topling/sideplugin-wiki-en/wiki/WebView) From 818d6def44dbabc68e10dd2bd62b2e665eb706c8 Mon Sep 17 00:00:00 2001 From: gydeng Date: Thu, 9 Oct 2025 03:14:09 +0000 Subject: [PATCH 29/51] chore(docs): add specs --- .specs/hugegraph-server/ToplingDB/design.md | 208 ++++++++++++++++++ .../ToplingDB/requirements.md | 35 +++ .specs/hugegraph-server/ToplingDB/task.md | 54 +++++ docs/{rocksdb-plus.md => toplingdb.md} | 54 ++--- 4 files changed, 324 insertions(+), 27 deletions(-) create mode 100644 .specs/hugegraph-server/ToplingDB/design.md create mode 100644 .specs/hugegraph-server/ToplingDB/requirements.md create mode 100644 .specs/hugegraph-server/ToplingDB/task.md rename docs/{rocksdb-plus.md => toplingdb.md} (61%) diff --git a/.specs/hugegraph-server/ToplingDB/design.md b/.specs/hugegraph-server/ToplingDB/design.md new file mode 100644 index 0000000000..db584147d4 --- /dev/null +++ b/.specs/hugegraph-server/ToplingDB/design.md @@ -0,0 +1,208 @@ +# Design of ToplingDB + +## Overview + +HugeGraph ToplingDB aims to enhance compatibility with ToplingDB, providing users with an additional storage engine option that improves performance, functionality, and usability. + +## Design Goals + +* **Dynamic Configuration**: Support flexible configuration of RocksDB parameters via YAML files, replacing hardcoded values to improve maintainability and adaptability. +* **Strong Compatibility**: Maintain full compatibility with the RocksDB API, ensuring seamless migration and integration with existing RocksDB code and data. +* **Visual Monitoring**: Provide a Web Server interface for real-time visibility into storage engine status and configuration, enhancing observability. +* **Simplified Deployment**: Automatically load dynamic libraries from JAR packages without requiring manual `LD_PRELOAD` setup or complex startup procedures, lowering the barrier for users. + +## Architecture Diagram + +#### HugeGraph Startup Script Logic + +Steps ❷ ~ ❻ in the diagram below illustrate the preload logic added to support ToplingDB. + +From the user's perspective, startup remains unchanged—simply execute `start-hugegraph.sh`. + +```mermaid +sequenceDiagram + autonumber + participant User as User + participant StartSh as start-hugegraph.sh + participant Preload as preload-topling.sh + participant JVM as JVM/LD Loader + participant Server as HugeGraph Server + + User->>StartSh: Execute startup script + StartSh->>Preload: Source preload script + Preload->>Preload: Locate rocksdbjni*.jar and extract .so to TOP/library + Preload->>Preload: Extract .html/.css to /dev/shm/rocksdb_resource + Preload->>JVM: Set LD_LIBRARY_PATH and LD_PRELOAD + Preload-->>StartSh: Return + StartSh->>Server: Start service + Server-->>User: Service running +``` + +#### RocksDB Startup Logic + +Use reflection to detect whether ToplingDB APIs are available. If present, attempt to start the storage engine using ToplingDB; otherwise, fall back to standard RocksDB APIs. + +```mermaid +sequenceDiagram + autonumber + participant Store as HugeGraph Store + participant Config as HugeConfig + participant Sessions as RocksDBStdSessions + participant SPR as SidePluginRepo (Reflection) + participant Repo as Repo Instance + participant Rocks as RocksDB + + Store->>Config: Read OPTION_PATH / OPEN_HTTP + Store->>Sessions: Create Sessions(..., optionPath, openHttp) + alt optionPath provided and SPR available + Sessions->>SPR: Reflectively load/create Repo + Sessions->>Repo: importAutoFile(optionPath) + Sessions->>Repo: open (with JSON descriptor) + opt openHttp is true and instance is GRAPH_STORE + Sessions->>Repo: startHttpServer() + end + Sessions->>Rocks: Get RocksDB instance (with CF handles) + Sessions-->>Store: Return OpenedRocksDB(repo, handles) + else optionPath not provided or SPR unavailable + Sessions->>Rocks: Standard open() + Sessions-->>Store: Return OpenedRocksDB(null, handles) + end + note over Store,Sessions: On shutdown, if repo exists, call repo.closeAllDB() +``` + +## Involved Modules + +### ToplingDB JAR and Maven Setup + +There are two ways to obtain the JAR package: + +1. Pull from GitHub repository +2. Build manually and install to Maven + +#### Pull JAR from GitHub Repository + +Since ToplingDB is not published to Maven Central, the JAR can only be obtained from GitHub Actions releases: +[JAR Package](https://github.com/hugegraph/toplingdb/packages/2550860) + +Add GitHub repository configuration to your Maven `settings.xml`: + +```xml + + github + https://maven.pkg.github.com/hugegraph/toplingdb + + true + + +``` + +Also, update the `rocksdbjni` version in `hugegraph-server/hugegraph-rocksdb/pom.xml` from `7.2.2` to `8.10.2-SNAPSHOT` to match the GitHub release: + +```xml + + org.rocksdb + rocksdbjni + 8.10.2-SNAPSHOT + +``` + +#### Build ToplingDB JAR Manually + +Clone [ToplingDB](https://github.com/topling/toplingdb) and run the following commands: + +```bash +# Build shared library +make -j$(nproc) DEBUG_LEVEL=0 shared_lib +# Install shared library +sudo make install-shared PREFIX=/opt DEBUG_LEVEL=0 +# Package JAR +make rocksdbjava -j$(nproc) DEBUG_LEVEL=0 STRIP_DEBUG_INFO=1 ROCKSDB_JAR_WITH_DYNAMIC_LIBS=1 + +# Set JAVA_HOME (especially for root) +export JAVA_HOME=/usr/lib/jvm/jre-openjdk-yourpath +# Install librocksdbjni dynamic library +sudo make -j install-jni PREFIX=/opt DEBUG_LEVEL=0 STRIP_DEBUG_INFO=1 +# Install JAR to local Maven repository +cd java/target +cp rocksdbjni-8.10.2-linux64.jar rocksdbjni-8.10.2-SNAPSHOT-linux64.jar +mvn install:install-file -Dfile=rocksdbjni-8.10.2-SNAPSHOT-linux64.jar \ + -DgroupId=org.rocksdb -DartifactId=rocksdbjni \ + -Dversion=8.10.2-SNAPSHOT -Dpackaging=jar +``` + +### Preloading Dynamic Libraries and Static Resources + +ToplingDB uses thread-local storage (TLS), requiring dynamic libraries to be preloaded via `LD_PRELOAD`. + +Additionally, the Web Server needs static resources to render the visualization interface. + +To support this, the `preload-topling.sh` script was added to preload ToplingDB dynamic libraries and Web Server resources. + +Main tasks of `preload-topling.sh`: + +- Extract `.so` libraries and Web resources (HTML/CSS) from `rocksdbjni*.jar` +- Set `LD_LIBRARY_PATH` and `LD_PRELOAD` environment variables +- Handle `libaio` compatibility issues on Ubuntu 24.04+ + +Both `init-hugegraph.sh` and `start-hugegraph.sh` now invoke `preload-topling.sh`, so users don’t need to worry about preload details. + +### HugeGraph Configuration Options for RocksDB + +Two new configuration options were added to `hugegraph.properties`: `option_path` for the YAML file and `open_http` to enable the Web Server. + +```properties +# rocksdb backend config +#rocksdb.data_path=/path/to/disk +#rocksdb.wal_path=/path/to/disk +#rocksdb.option_path=./conf/graphs/rocksdb_plus.yaml +#rocksdb.open_http=true +``` + +Java-side parsing and default values: + +```java +public static final ConfigOption OPTION_PATH = + new ConfigOption<>( + "rocksdb.option_path", + "The YAML file for configuring Topling/RocksDB parameters", + null, + "" + ); + +public static final ConfigOption OPEN_HTTP = + new ConfigOption<>( + "rocksdb.open_http", + "Whether to start Topling's HTTP service", + disallowEmpty(), + false + ); +``` + +### ToplingDB Startup Logic + +When `option_path` is configured and the JAR contains ToplingDB APIs, HugeGraph will load the YAML file and start ToplingDB. Otherwise, it falls back to standard RocksDB. + +To keep port configuration simple, the Web Server is only enabled for the `GRAPH_STORE` instance. + +```mermaid +flowchart LR + A[Start Initialization] --> B{Is optionPath provided?} + + B -- No --> Z[Use standard RocksDB to open DB] --> O[Finish standard init] + + B -- Yes --> C{Is SidePluginRepo class available?} + C -- No --> Z + + C -- Yes --> D[Set useTopling = true] --> E["Load SidePluginRepo class and call importAutoFile(optionPath)"] + + E --> K{Is openHttp true?} + K -- No --> M[Finish Topling init] + K -- Yes --> L{Is dbName GRAPH_STORE?} + L -- No --> M + L -- Yes --> N["startHttpServer()"] --> M +``` + +## Design Decisions and Rationale + +1. **Why is the Web Server only started for GRAPH_STORE?** + - All graph data is stored in GRAPH_STORE, and performance tuning and observability are primarily focused on this instance. diff --git a/.specs/hugegraph-server/ToplingDB/requirements.md b/.specs/hugegraph-server/ToplingDB/requirements.md new file mode 100644 index 0000000000..6530b32cae --- /dev/null +++ b/.specs/hugegraph-server/ToplingDB/requirements.md @@ -0,0 +1,35 @@ +# Requirements of ToplingDB + +## Introduction + +RocksDB is the primary standalone/distributed backend storage engine planned for HugeGraph. +However, the current `rocksdb-jni` design makes it difficult for HugeGraph to dynamically modify or adjust RocksDB parameters, resulting in limited flexibility and extensive hard-coding logic. + +To improve performance, functionality, and usability, HugeGraph introduces `ToplingDB` as an optional enhancement. +It allows users to configure RocksDB parameters via external configuration files and visualize storage engine status through a built-in Web Server. + +## Requirement List + +### 1. Support ToplingDB while maintaining compatibility with RocksDB + +**User Story**: As a long-term user, I want the system to support the enhanced features of ToplingDB without affecting existing RocksDB functionality or data compatibility. + +**Acceptance Criteria**: Users can choose between RocksDB and ToplingDB via configuration files or startup parameters. + +### 2. Support configuring RocksDB parameters via external configuration files + +**User Story**: As a user, I want to adjust storage engine parameters based on my business needs and hardware environment to optimize database performance. + +**Acceptance Criteria**: The system supports passing configuration files to customize RocksDB parameters. + +### 3. Support runtime observability of the RocksDB storage engine + +**User Story**: As a system operator, I want clear and intuitive visibility into RocksDB configuration and runtime status. + +**Acceptance Criteria**: The system supports enhancing storage engine observability via a Web Server. + +## Success Criteria + +* The system maintains API compatibility with both RocksDB and ToplingDB. +* The system supports configuring ToplingDB parameters via external configuration files. +* The system enhances storage engine observability through a built-in Web Server. diff --git a/.specs/hugegraph-server/ToplingDB/task.md b/.specs/hugegraph-server/ToplingDB/task.md new file mode 100644 index 0000000000..c68a658530 --- /dev/null +++ b/.specs/hugegraph-server/ToplingDB/task.md @@ -0,0 +1,54 @@ +# Tasks of ToplingDB + +This document translates the design of HugeGraph ToplingDB into a series of executable development tasks. Each task follows a test-driven approach to ensure incremental progress and early validation. + +## Common Development Commands + +### Runtime + +ToplingDB (ToplingDB) requires dynamic libraries to be preloaded via `LD_PRELOAD`. The `preload-topling.sh` script parses the JAR package, extracts the necessary libraries, and performs the preload setup. + +When using an IDE such as IntelliJ IDEA, you need to configure the following environment variables in Run/Debug Configurations: + +```shell +LD_LIBRARY_PATH=/path/to/your/library:$LD_LIBRARY_PATH +LD_PRELOAD=libjemalloc.so:librocksdbjni-linux64.so +``` + +When running from the terminal, simply use `init-store.sh` and `start-hugegraph.sh`, as `preload-topling.sh` is already embedded in these scripts. + +## 1. Project Infrastructure Setup + +- [x] **1.1 Build ToplingDB JAR Package** + - Publish the package to GitHub Packages via GitHub Actions and update Maven's `settings.xml` + - Provide documentation for manually building the ToplingDB JAR package + +## 2. Compatibility with ToplingDB and Standard RocksDB + +- [x] **2.1 Modify openRocksDB logic in RocksDBStdSession** + - Use reflection to detect whether the current JAR contains ToplingDB APIs; if so, start the storage engine using ToplingDB + - If not available, fall back to the standard RocksDB API for engine startup + +## 3. Add Configuration Options for ToplingDB in HugeGraph + +- [x] **3.1 Add `rocksdb.option_path` configuration** + - Type: string, used to specify the path to the YAML configuration file + - Allow users to pass the YAML file via `hugegraph.properties` using `rocksdb.option_path` + - This option is invalid for standard RocksDB JARs, as RocksDB APIs do not support file-based configuration + +- [x] **3.2 Add `rocksdb.open_http` configuration** + - Type: boolean, used to specify whether to enable the ToplingDB Web Server + - Allow users to configure Web Server activation via `rocksdb.open_http` in `hugegraph.properties` + - The Web Server port is defined in the YAML file specified by `option_path`, under `http.listening_ports` + - For simplicity, the Web Server is only enabled for the `GRAPH_STORE` instance that stores graph data + +## 4. End-to-End Performance Testing + +- [x] **4.1 Write Performance Testing** + - Use `hugegraph-loader` to load the twitter-2010 dataset + - Shuffle the twitter-2010 dataset to simulate real-world random insertion patterns and evaluate the write performance of ToplingDB. + - ToplingDB improves random write performance by up to 40% and reduces storage overhead by approximately 50% + +- [x] **4.2 Read Performance Testing** + - Execute edge traversal, vertex traversal, and KOUT queries to evaluate read performance improvements + - Under cold start conditions, edge traversal latency is reduced by up to 50%, and KOUT query average latency is reduced by approximately 15% diff --git a/docs/rocksdb-plus.md b/docs/toplingdb.md similarity index 61% rename from docs/rocksdb-plus.md rename to docs/toplingdb.md index 93045a8f03..b5941f39ab 100644 --- a/docs/rocksdb-plus.md +++ b/docs/toplingdb.md @@ -1,4 +1,4 @@ -# RocksDB Plus (ToplingDB) Support and Configuration +# ToplingDB Support and Configuration - **Status**: Implemented - **Pull Request**: [#15](https://github.com/hugegraph/hugegraph/pull/15) @@ -21,38 +21,38 @@ ToplingDB extends RocksDB with several advanced features: ## Motivation -Introduce a new optional storage component in HugeGraph to support RocksDB Plus ([ToplingDB](https://github.com/topling/toplingdb)), a configurable and observable extension of the RocksDB storage engine. +Introduce a new optional storage component in HugeGraph to support [ToplingDB](https://github.com/topling/toplingdb)), a configurable and observable extension of the RocksDB storage engine. -RocksDB Plus resolves key limitations in HugeGraph’s current `rocksdbjni` integration, which relies heavily on hard-coding parameters and lacks runtime configurability and observability. +ToplingDB resolves key limitations in HugeGraph’s current `rocksdbjni` integration, which relies heavily on hard-coding parameters and lacks runtime configurability and observability. -By enabling YAML-based configuration and exposing a Web Server interface, RocksDB Plus allows users to fine-tune performance and monitor engine behavior without modifying code or restarting services. +By enabling YAML-based configuration and exposing a Web Server interface, ToplingDB allows users to fine-tune performance and monitor engine behavior without modifying code or restarting services. This is especially valuable in environments where storage workloads vary across deployments, and where operational transparency is critical for debugging and optimization. For example, in production clusters with heterogeneous hardware or mixed graph workloads, users can adjust compaction, caching, and I/O settings to match their performance goals. -Additionally, RocksDB Plus maintains full compatibility with the existing RocksDB API, allowing seamless migration and fallback. Users can opt into RocksDB Plus via configuration, without impacting legacy data or workflows. +Additionally, ToplingDB maintains full compatibility with the existing RocksDB API, allowing seamless migration and fallback. Users can opt into ToplingDB via configuration, without impacting legacy data or workflows. -By supporting RocksDB Plus, HugeGraph empowers users with greater control over storage behavior, simplifies deployment through automated dynamic library loading, and enhances operational insight—all while preserving compatibility and ease of use. +By supporting ToplingDB, HugeGraph empowers users with greater control over storage behavior, simplifies deployment through automated dynamic library loading, and enhances operational insight—all while preserving compatibility and ease of use. ## Goals -**Introduce RocksDB Plus as a configurable and observable alternative to RocksDB.** +**Introduce ToplingDB as a configurable and observable alternative to RocksDB.** -Enable users to select RocksDB Plus via configuration, allowing tuning parameters through YAML files without recompilation and real-time monitoring via Web Server—without sacrificing compatibility with existing RocksDB APIs. +Enable users to select ToplingDB via configuration, allowing tuning parameters through YAML files without recompilation and real-time monitoring via Web Server—without sacrificing compatibility with existing RocksDB APIs. ## Design ### Configuration Parameters -To support RocksDB Plus in HugeGraph, two new configuration parameters have been introduced: `rocksdb.option_path` and `rocksdb.open_http`. These options allow users to configure RocksDB parameters and enable real-time observability. +To support ToplingDB in HugeGraph, two new configuration parameters have been introduced: `rocksdb.option_path` and `rocksdb.open_http`. These options allow users to configure RocksDB parameters and enable real-time observability. #### `rocksdb.option_path`: External YAML Configuration -This parameter allows users to specify a YAML file that defines RocksDB Plus settings such as compaction strategy, cache size, compression type, and more. +This parameter allows users to specify a YAML file that defines ToplingDB settings such as compaction strategy, cache size, compression type, and more. - **Purpose**: Replace hard-coding parameters with flexible, file-based configuration. @@ -62,19 +62,19 @@ This parameter allows users to specify a YAML file that defines RocksDB Plus set rocksdb.option_path=./conf/graphs/rocksdb_plus.yaml ``` - The specified YAML file will be automatically loaded during database initialization if RocksDB Plus is available. + The specified YAML file will be automatically loaded during database initialization if ToplingDB is available. For details on the YAML structure and supported configuration fields, please refer to [SidePlugin](https://github.com/topling/sideplugin-wiki-en/wiki). -- **Implementation**: During initialization, HugeGraph checks whether the configured JAR contains RocksDB Plus APIs. If so, it uses reflection to load the SidePluginRepo class and calls `importAutoFile(optionPath)` to parse the YAML file. The resulting configuration is applied to the RocksDB instance. +- **Implementation**: During initialization, HugeGraph checks whether the configured JAR contains ToplingDB APIs. If so, it uses reflection to load the SidePluginRepo class and calls `importAutoFile(optionPath)` to parse the YAML file. The resulting configuration is applied to the RocksDB instance. -- **Fallback**: If the YAML file is not provided or RocksDB Plus is unavailable, HugeGraph will fall back to standard RocksDB behavior. +- **Fallback**: If the YAML file is not provided or ToplingDB is unavailable, HugeGraph will fall back to standard RocksDB behavior. #### `rocksdb.open_http`: Enable Web Server for Observability -This boolean flag controls whether the embedded Web Server in RocksDB Plus should be started. The server exposes runtime metrics, configuration status, and internal RocksDB statistics via a browser-accessible interface. +This boolean flag controls whether the embedded Web Server in ToplingDB should be started. The server exposes runtime metrics, configuration status, and internal RocksDB statistics via a browser-accessible interface. - **Purpose**: Provide real-time visibility into the storage engine for debugging and performance tuning. @@ -93,7 +93,7 @@ This boolean flag controls whether the embedded Web Server in RocksDB Plus shoul To preview the Web Server interface and its layout, see [Web Server](https://github.com/topling/sideplugin-wiki-en/wiki/WebView). -- **Implementation**: If `open_http` is set to true and the database instance is `GRAPH_STORE`, HugeGraph invokes `startHttpServer()` on the RocksDB Plus repo object. This exposes a browser-accessible dashboard for monitoring RocksDB internals. +- **Implementation**: If `open_http` is set to true and the database instance is `GRAPH_STORE`, HugeGraph invokes `startHttpServer()` on the ToplingDB repo object. This exposes a browser-accessible dashboard for monitoring RocksDB internals. - **Scope**: For simplicity, the Web Server is only enabled for the `GRAPH_STORE` instance, which holds the main graph data. @@ -102,19 +102,19 @@ This boolean flag controls whether the embedded Web Server in RocksDB Plus shoul ### Reflection-Based Loading Mechanism -To support RocksDB Plus without introducing hard dependencies, HugeGraph uses Java reflection to detect and load enhanced APIs at runtime. +To support ToplingDB without introducing hard dependencies, HugeGraph uses Java reflection to detect and load enhanced APIs at runtime. -During initialization, HugeGraph checks whether the current JAR contains the class `com.topling.sideplugin.SidePluginRepo`. If present, it assumes RocksDB Plus is available and proceeds to: +During initialization, HugeGraph checks whether the current JAR contains the class `com.topling.sideplugin.SidePluginRepo`. If present, it assumes ToplingDB is available and proceeds to: 1. **Load the SidePluginRepo class via reflection** This avoids compile-time coupling and allows fallback to standard RocksDB if the class is missing. - * If the RocksDB Plus API cannot be found, HugeGraph silently falls back to the standard RocksDB API for startup. + * If the ToplingDB API cannot be found, HugeGraph silently falls back to the standard RocksDB API for startup. 2. **Invoke** `importAutoFile(optionPath)` This method parses the YAML configuration file specified by `rocksdb.option_path` to configure storage engine parameters. - * If the `option_path` is incorrect or parsing fails, RocksDB Plus throws an error and terminates the startup process. -3. **Call** `open()` **with a JSON descriptor** The parsed configuration is converted to a JSON structure and passed to the RocksDB Plus engine to initialize the database. + * If the `option_path` is incorrect or parsing fails, ToplingDB throws an error and terminates the startup process. +3. **Call** `open()` **with a JSON descriptor** The parsed configuration is converted to a JSON structure and passed to the ToplingDB engine to initialize the database. 4. **Optionally start the Web Server** If `rocksdb.open_http` is true and the instance is `GRAPH_STORE`, HugeGraph invokes `startHttpServer()` via reflection to enable observability. - * If the Web Server cannot be started due to misconfiguration **or if the specified HTTP port is already in use**, RocksDB Plus throws an error and the startup process is terminated + * If the Web Server cannot be started due to misconfiguration **or if the specified HTTP port is already in use**, ToplingDB throws an error and the startup process is terminated -This design ensures that RocksDB Plus can be integrated as an optional enhancement, without breaking compatibility or requiring changes to the core HugeGraph codebase. +This design ensures that ToplingDB can be integrated as an optional enhancement, without breaking compatibility or requiring changes to the core HugeGraph codebase. @@ -122,15 +122,15 @@ This design ensures that RocksDB Plus can be integrated as an optional enhanceme ### For Users -The way users operate remains unchanged by default, and adding RocksDB Plus configuration provides additional functionality. -The RocksDB Plus integration is fully embedded into the existing startup scripts (`init-store.sh` and `start-hugegraph.sh`). Users only need to set `rocksdb.option_path` to specify the YAML file path and adjust its contents as needed to tune the storage engine. +The way users operate remains unchanged by default, and adding ToplingDB configuration provides additional functionality. +The ToplingDB integration is fully embedded into the existing startup scripts (`init-store.sh` and `start-hugegraph.sh`). Users only need to set `rocksdb.option_path` to specify the YAML file path and adjust its contents as needed to tune the storage engine. ### For Developers -Developers need to make two adjustments to enable RocksDB Plus during development: +Developers need to make two adjustments to enable ToplingDB during development: -1. **Maven Repository Configuration**: since RocksDB Plus is published via GitHub Packages, developers must add the GitHub repository to their `settings.xml` to fetch the correct JAR: +1. **Maven Repository Configuration**: since ToplingDB is published via GitHub Packages, developers must add the GitHub repository to their `settings.xml` to fetch the correct JAR: ```xml @@ -177,7 +177,7 @@ Developers need to make two adjustments to enable RocksDB Plus during developmen LD_PRELOAD="libjemalloc.so:librocksdbjni.so" ``` -These steps ensure that RocksDB Plus loads correctly in development environments and behaves consistently with production deployments. +These steps ensure that ToplingDB loads correctly in development environments and behaves consistently with production deployments. From aa477e0b25d2a2104b8cf7b7c11f35f370d1e6fe Mon Sep 17 00:00:00 2001 From: gydeng Date: Thu, 9 Oct 2025 03:20:49 +0000 Subject: [PATCH 30/51] chore(docs): fix lint --- .specs/hugegraph-server/ToplingDB/design.md | 4 +-- .specs/hugegraph-server/ToplingDB/task.md | 32 ++++++++++----------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.specs/hugegraph-server/ToplingDB/design.md b/.specs/hugegraph-server/ToplingDB/design.md index db584147d4..1d810df84d 100644 --- a/.specs/hugegraph-server/ToplingDB/design.md +++ b/.specs/hugegraph-server/ToplingDB/design.md @@ -13,7 +13,7 @@ HugeGraph ToplingDB aims to enhance compatibility with ToplingDB, providing user ## Architecture Diagram -#### HugeGraph Startup Script Logic +### HugeGraph Startup Script Logic Steps ❷ ~ ❻ in the diagram below illustrate the preload logic added to support ToplingDB. @@ -38,7 +38,7 @@ sequenceDiagram Server-->>User: Service running ``` -#### RocksDB Startup Logic +### RocksDB Startup Logic Use reflection to detect whether ToplingDB APIs are available. If present, attempt to start the storage engine using ToplingDB; otherwise, fall back to standard RocksDB APIs. diff --git a/.specs/hugegraph-server/ToplingDB/task.md b/.specs/hugegraph-server/ToplingDB/task.md index c68a658530..7ae83eff1b 100644 --- a/.specs/hugegraph-server/ToplingDB/task.md +++ b/.specs/hugegraph-server/ToplingDB/task.md @@ -20,35 +20,35 @@ When running from the terminal, simply use `init-store.sh` and `start-hugegraph. ## 1. Project Infrastructure Setup - [x] **1.1 Build ToplingDB JAR Package** - - Publish the package to GitHub Packages via GitHub Actions and update Maven's `settings.xml` - - Provide documentation for manually building the ToplingDB JAR package + - Publish the package to GitHub Packages via GitHub Actions and update Maven's `settings.xml` + - Provide documentation for manually building the ToplingDB JAR package ## 2. Compatibility with ToplingDB and Standard RocksDB - [x] **2.1 Modify openRocksDB logic in RocksDBStdSession** - - Use reflection to detect whether the current JAR contains ToplingDB APIs; if so, start the storage engine using ToplingDB - - If not available, fall back to the standard RocksDB API for engine startup + - Use reflection to detect whether the current JAR contains ToplingDB APIs; if so, start the storage engine using ToplingDB + - If not available, fall back to the standard RocksDB API for engine startup ## 3. Add Configuration Options for ToplingDB in HugeGraph - [x] **3.1 Add `rocksdb.option_path` configuration** - - Type: string, used to specify the path to the YAML configuration file - - Allow users to pass the YAML file via `hugegraph.properties` using `rocksdb.option_path` - - This option is invalid for standard RocksDB JARs, as RocksDB APIs do not support file-based configuration + - Type: string, used to specify the path to the YAML configuration file + - Allow users to pass the YAML file via `hugegraph.properties` using `rocksdb.option_path` + - This option is invalid for standard RocksDB JARs, as RocksDB APIs do not support file-based configuration - [x] **3.2 Add `rocksdb.open_http` configuration** - - Type: boolean, used to specify whether to enable the ToplingDB Web Server - - Allow users to configure Web Server activation via `rocksdb.open_http` in `hugegraph.properties` - - The Web Server port is defined in the YAML file specified by `option_path`, under `http.listening_ports` - - For simplicity, the Web Server is only enabled for the `GRAPH_STORE` instance that stores graph data + - Type: boolean, used to specify whether to enable the ToplingDB Web Server + - Allow users to configure Web Server activation via `rocksdb.open_http` in `hugegraph.properties` + - The Web Server port is defined in the YAML file specified by `option_path`, under `http.listening_ports` + - For simplicity, the Web Server is only enabled for the `GRAPH_STORE` instance that stores graph data ## 4. End-to-End Performance Testing - [x] **4.1 Write Performance Testing** - - Use `hugegraph-loader` to load the twitter-2010 dataset - - Shuffle the twitter-2010 dataset to simulate real-world random insertion patterns and evaluate the write performance of ToplingDB. - - ToplingDB improves random write performance by up to 40% and reduces storage overhead by approximately 50% + - Use `hugegraph-loader` to load the twitter-2010 dataset + - Shuffle the twitter-2010 dataset to simulate real-world random insertion patterns and evaluate the write performance of ToplingDB. + - ToplingDB improves random write performance by up to 40% and reduces storage overhead by approximately 50% - [x] **4.2 Read Performance Testing** - - Execute edge traversal, vertex traversal, and KOUT queries to evaluate read performance improvements - - Under cold start conditions, edge traversal latency is reduced by up to 50%, and KOUT query average latency is reduced by approximately 15% + - Execute edge traversal, vertex traversal, and KOUT queries to evaluate read performance improvements + - Under cold start conditions, edge traversal latency is reduced by up to 50%, and KOUT query average latency is reduced by approximately 15% From d632118ed693eff0acd84b9e1351219fb1366b27 Mon Sep 17 00:00:00 2001 From: gydeng Date: Thu, 9 Oct 2025 03:26:54 +0000 Subject: [PATCH 31/51] chore(docs): update maven settings.xml conf --- .specs/hugegraph-server/ToplingDB/design.md | 33 ++++++++++++++++----- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/.specs/hugegraph-server/ToplingDB/design.md b/.specs/hugegraph-server/ToplingDB/design.md index 1d810df84d..c589ee9be2 100644 --- a/.specs/hugegraph-server/ToplingDB/design.md +++ b/.specs/hugegraph-server/ToplingDB/design.md @@ -87,13 +87,32 @@ Since ToplingDB is not published to Maven Central, the JAR can only be obtained Add GitHub repository configuration to your Maven `settings.xml`: ```xml - - github - https://maven.pkg.github.com/hugegraph/toplingdb - - true - - + + + + + github + YOUR_GITHUB_ACTOR + YOUR_GITHUB_TOKEN + + + + + + ... + + ... + + + github + https://maven.pkg.github.com/hugegraph/toplingdb + + true + + + + + ``` Also, update the `rocksdbjni` version in `hugegraph-server/hugegraph-rocksdb/pom.xml` from `7.2.2` to `8.10.2-SNAPSHOT` to match the GitHub release: From 6c707d909cd73f544e52573946f66afc7dff4cb0 Mon Sep 17 00:00:00 2001 From: gydeng Date: Thu, 9 Oct 2025 06:40:12 +0000 Subject: [PATCH 32/51] chore(docs): add token permission hint --- .specs/hugegraph-server/ToplingDB/design.md | 1 + docs/toplingdb.md | 1 + 2 files changed, 2 insertions(+) diff --git a/.specs/hugegraph-server/ToplingDB/design.md b/.specs/hugegraph-server/ToplingDB/design.md index c589ee9be2..4a4cc378b6 100644 --- a/.specs/hugegraph-server/ToplingDB/design.md +++ b/.specs/hugegraph-server/ToplingDB/design.md @@ -93,6 +93,7 @@ Add GitHub repository configuration to your Maven `settings.xml`: github YOUR_GITHUB_ACTOR + YOUR_GITHUB_TOKEN diff --git a/docs/toplingdb.md b/docs/toplingdb.md index b5941f39ab..633d0660fb 100644 --- a/docs/toplingdb.md +++ b/docs/toplingdb.md @@ -139,6 +139,7 @@ Developers need to make two adjustments to enable ToplingDB during development: github YOUR_GITHUB_ACTOR + YOUR_GITHUB_TOKEN From 668c5a36c874766fff466770a745d3afd2bbd904 Mon Sep 17 00:00:00 2001 From: gydeng Date: Wed, 15 Oct 2025 04:27:37 +0000 Subject: [PATCH 33/51] fix: add YAML file validation --- .github/workflows/server-ci.yml | 6 +- docs/toplingdb/toplingdb-operations.md | 87 +++++++++++ docs/toplingdb/toplingdb-security.md | 50 +++++++ docs/toplingdb/toplingdb-troubleshooting.md | 138 ++++++++++++++++++ docs/{ => toplingdb}/toplingdb.md | 33 ++--- .../static/conf/graphs/rocksdb_plus.yaml | 2 +- .../backend/store/rocksdb/RocksDBOptions.java | 77 ++++++++++ .../store/rocksdb/RocksDBStdSessions.java | 4 + 8 files changed, 367 insertions(+), 30 deletions(-) create mode 100644 docs/toplingdb/toplingdb-operations.md create mode 100644 docs/toplingdb/toplingdb-security.md create mode 100644 docs/toplingdb/toplingdb-troubleshooting.md rename docs/{ => toplingdb}/toplingdb.md (95%) diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index 08eee3b0d1..48f59b7d01 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -72,13 +72,9 @@ jobs: cp $HOME/.m2/settings.xml /tmp/settings.xml cp -vf .github/configs/settings.xml $HOME/.m2/settings.xml && cat $HOME/.m2/settings.xml - - name: Compile - run: | - mvn clean compile -U -Dmaven.javadoc.skip=true -ntp - - name: Package run: | - mvn package -Dmaven.test.skip=true -ntp + mvn clean package -Dmaven.test.skip=true -ntp source $TRAVIS_DIR/install-rocksdb.sh - name: Prepare backend environment diff --git a/docs/toplingdb/toplingdb-operations.md b/docs/toplingdb/toplingdb-operations.md new file mode 100644 index 0000000000..b5e3ea9055 --- /dev/null +++ b/docs/toplingdb/toplingdb-operations.md @@ -0,0 +1,87 @@ +# ToplingDB Operations Guide + +This guide outlines key operational practices for deploying, monitoring, tuning, and upgrading ToplingDB in production environments. It is intended for system administrators, DevOps engineers, and database maintainers seeking to ensure stability, performance, and scalability. + +--- + +## Monitoring Metrics + +- **Key Performance Indicators (KPI)** + - Write throughput (bytes/sec, ops/sec) + - Read latency (P95/P99) + - MemTable usage and flush frequency + - Block cache hit ratio + +- **Alert Thresholds** + - L0 file count exceeding `level0_stop_writes_trigger` + - Background job saturation (`max_background_jobs`) + - WAL size growth beyond expected limits + - Cache eviction rate anomalies + +- **Monitoring Tool Integration** + - Export metrics via HTTP endpoints + - Integrate with Prometheus using custom exporters + - Visualize trends and thresholds in Grafana dashboards + - Use SidePlugin’s web server (`listening_ports`) for live inspection + +--- + +## Performance Tuning + +- **Cache Size Optimization** + - Adjust `capacity` in `lru_cache` based on workload and memory budget + - Tune `high_pri_pool_ratio` to prioritize index/filter caching + +- **Compression Algorithm Selection** + - Use `kSnappyCompression` for balanced speed and space + - Disable compression (`kNoCompression`) for latency-sensitive workloads + +- **Compaction Strategy Adjustment** + - Set `level0_file_num_compaction_trigger` to control L0 flush frequency + - Use `level_compaction_dynamic_file_size: true` to adapt SST sizing + - Tune `max_subcompactions` and `max_background_jobs` for parallelism + +- **I/O Tuning Parameters** + - Evaluate `convert_to_sst: kFileMmap` to bypass traditional flush + - Set `sync_sst_file: false` for performance, with caution on durability + - Adjust `compaction_readahead_size` for sequential disk access + +--- + +## Capacity Planning + +- **Disk Space Estimation** + - Base on `write_buffer_size`, `target_file_size_base`, and compaction amplification + - Include space for WAL, MANIFEST, and temporary files + +- **Memory Requirement Calculation** + - Sum of MemTable (`mem_cap`), block cache (`capacity`), and background buffers + - Consider `max_write_buffer_number` and `min_write_buffer_number_to_merge` + +- **CPU Resource Planning** + - Allocate cores for compaction (`max_background_compactions`) + - Reserve CPU for Hugegraph query threads and SidePlugin HTTP services + - Monitor mutex contention (`use_adaptive_mutex`) and shard parallelism + +--- + +## Upgrade Procedure + +- **Version Compatibility Check** + - Review changelogs and YAML schema changes + - Validate plugin compatibility (e.g., `cspp`, `DispatcherTable`) + +- **Data Backup Strategy** + - Snapshot SST files and MANIFEST + - Backup column family metadata and configuration files + +- **Rolling Upgrade Steps** + - Drain traffic from target node + - Stop ToplingDB process and apply new binary + - Validate startup with `create_if_missing: false` + - Rejoin cluster and monitor metrics + +- **Rollback Plan** + - Restore previous binary and configuration + - Revert SST and MANIFEST from backup + - Disable incompatible plugins if needed diff --git a/docs/toplingdb/toplingdb-security.md b/docs/toplingdb/toplingdb-security.md new file mode 100644 index 0000000000..e57b877426 --- /dev/null +++ b/docs/toplingdb/toplingdb-security.md @@ -0,0 +1,50 @@ +# ToplingDB Security Hardening Guide + +This document provides best practices for securing a ToplingDB deployment. It covers file permissions, network access control, firewall rules, and additional hardening measures to reduce the attack surface and ensure safe operation in production environments. + +--- + +## 1. File Permissions + +Restrict file permissions to prevent unauthorized access or modification of scripts and configuration files: + +```bash +chmod 750 $HUGEGRAPH_HOME/bin/*.sh +chmod 640 $HUGEGRAPH_HOME/conf/graphs/*.yaml +chown -R hugegraph:hugegraph $HUGEGRAPH_HOME +``` + +- `750` ensures only the owner can execute scripts, while group members can read them. +- `640` ensures configuration files are readable by the owner and group, but not world-readable. +- Ownership should be assigned to a dedicated service account (e.g., `hugegraph`). + +--- + +## 2. Network Access Control + +Restrict network exposure by binding services to localhost or specific interfaces: + +```yaml +# Localhost-only access +http: + listening_ports: '127.0.0.1:2011' +``` + +- Avoid binding to `0.0.0.0` unless absolutely necessary. +- Use reverse proxies (e.g., Nginx) or VPN tunnels if remote access is required. + +--- + +## 3. Firewall Rules + +Use firewall rules to limit access to trusted IP ranges: + +```bash +# Allow only specific subnet to access the Web Server +iptables -A INPUT -p tcp --dport 2011 \ + -s 192.168.1.0/24 -j ACCEPT +iptables -A INPUT -p tcp --dport 2011 -j DROP +``` + +- Replace `192.168.1.0/24` with your trusted network. +- Consider using `firewalld` or `ufw` for simplified management. diff --git a/docs/toplingdb/toplingdb-troubleshooting.md b/docs/toplingdb/toplingdb-troubleshooting.md new file mode 100644 index 0000000000..68d8072c46 --- /dev/null +++ b/docs/toplingdb/toplingdb-troubleshooting.md @@ -0,0 +1,138 @@ +# ToplingDB Troubleshooting + +## Issues + +### Issue 1: Startup Failure Due to YAML Format Error + +Sample log output: + +```java +2025-10-15 01:55:50 [db-open-1] [INFO] o.a.h.b.s.r.RocksDBStdSessions - SidePluginRepo found. Will attempt to open multi CFs RocksDB using Topling plugin. +21:1: (891B):ERROR: +sideplugin/rockside/3rdparty/rapidyaml/src/c4/yml/parse.cpp:3310: ERROR parsing yml: parse error: incorrect indentation? +``` + +**Solution**: + +1. Check that YAML indentation is correct (must use spaces, not tabs). +2. Validate YAML syntax: + + ```bash + python -c "import yaml; yaml.safe_load(open('conf/graphs/rocksdb_plus.yaml'))" + ``` + +3. Review the specific error message in the logs for further clues. + +--- + +### Issue 2: Web Server Port Conflict + +Sample log output: + +```java +2025-10-15 01:57:34 [db-open-1] [INFO] o.a.h.b.s.r.RocksDBStdSessions - SidePluginRepo found. Will attempt to open multi CFs RocksDB using Topling plugin. +2025-10-15 01:57:34 [db-open-1] [ERROR] o.a.h.b.s.r.RocksDBStore - Failed to open RocksDB 'rocksdb-data/data/g' +org.rocksdb.RocksDBException: rocksdb::Status rocksdb::SidePluginRepo::StartHttpServer(): null context when constructing CivetServer. Possible problem binding to port. + at org.rocksdb.SidePluginRepo.startHttpServer(Native Method) ~[rocksdbjni-8.10.2-20250804.074027-4.jar:?] +``` + +**Solution**: + +1. Check if the port is already in use: + + ```bash + lsof -i :2011 + ``` + +2. Modify the `listening_ports` setting in the YAML configuration file. +3. Restart the HugeGraph Server. + +--- + +### Issue 3: Database Initialization Failure + +This error indicates the database lock file cannot be acquired, possibly due to insufficient write permissions or another process holding the lock: + +```java +Caused by: org.rocksdb.RocksDBException: While lock file: rocksdb-data/data/m/LOCK: Resource temporarily unavailable + at org.rocksdb.SidePluginRepo.nativeOpenDBMultiCF(Native Method) + at org.rocksdb.SidePluginRepo.openDB(SidePluginRepo.java:22) +``` + +**Solution**: + +1. Confirm the configuration file path is correct: + + ```properties + rocksdb.option_path=./conf/graphs/rocksdb_plus.yaml + ``` + +2. Check permissions on the data directory to ensure the running user has read/write access. +3. Review detailed logs: + + ```bash + bin/init-store.sh 2>&1 | tee init.log + ``` + +--- + +## Log Analysis + +### Enable Debug Logging + +```properties +# conf/log4j2.xml + +``` + +### Key Log Locations + +- Application logs: `logs/hugegraph-server.log` +- RocksDB logs: `data/rocksdb/LOG` +- Web Server logs: check the `access_log` setting in the YAML configuration + +Additional notes: + +- Enable debug logging only during troubleshooting, as it may generate large log files and impact performance. +- Rotate and archive logs regularly to prevent disk space exhaustion. + +--- + +## Performance Diagnostics + +### High CPU Usage + +1. Review and tune **compaction** configuration (e.g., compaction style, trigger thresholds). +2. Adjust **thread pool size** to match available CPU cores and workload characteristics. +3. Optimize **write batching** to reduce per-operation overhead. +4. Monitor for **hot keys** or skewed workloads that may cause uneven CPU usage. +5. Use performance profiling tools (e.g., `perf`, `async-profiler`) to identify hotspots. + +--- + +### Excessive Memory Usage + +1. Adjust **block cache size** to balance between read performance and memory footprint. +2. Review **write buffer** (memtable) configuration, including number and size. +3. Monitor for **memory leaks** in the application layer or plugins. +4. Enable **JVM GC logging** to analyze garbage collection behavior. + +--- + +### Disk I/O Bottlenecks + +1. Use **SSD storage** for RocksDB data directories to improve latency and throughput. +2. Tune **WAL (Write-Ahead Log)** configuration, such as enabling `wal_dir` on a separate disk. +3. Optimize **compaction strategy** (e.g., level-based vs. universal compaction) based on workload. +4. Monitor **disk utilization** and IOPS using tools like `iostat` or `dstat`. +5. Separate **data, WAL, and log directories** onto different physical devices if possible. + +--- + +### General Recommendations + +- Always benchmark configuration changes in a staging environment before applying them to production. +- Use monitoring systems (e.g., Prometheus + Grafana) to track CPU, memory, and I/O metrics over time. +- Regularly review RocksDB’s internal statistics (`rocksdb.stats`) for deeper insights into performance. +- Automate log collection and alerting to quickly detect anomalies. diff --git a/docs/toplingdb.md b/docs/toplingdb/toplingdb.md similarity index 95% rename from docs/toplingdb.md rename to docs/toplingdb/toplingdb.md index 633d0660fb..09bdcf787d 100644 --- a/docs/toplingdb.md +++ b/docs/toplingdb/toplingdb.md @@ -3,8 +3,6 @@ - **Status**: Implemented - **Pull Request**: [#15](https://github.com/hugegraph/hugegraph/pull/15) - - ## Background knowledge [ToplingDB](https://github.com/topling/toplingdb) is a high-performance, cloud-native key-value store built as a fork of RocksDB. @@ -17,8 +15,6 @@ ToplingDB extends RocksDB with several advanced features: - **Distributed Compaction**: Designed for cloud environments, ToplingDB supports distributed compaction strategies to reduce write amplification and improve throughput. - **Compatibility**: Drop-in replacement for RocksDB in most use cases. - - ## Motivation Introduce a new optional storage component in HugeGraph to support [ToplingDB](https://github.com/topling/toplingdb)), a configurable and observable extension of the RocksDB storage engine. @@ -35,15 +31,12 @@ Additionally, ToplingDB maintains full compatibility with the existing RocksDB A By supporting ToplingDB, HugeGraph empowers users with greater control over storage behavior, simplifies deployment through automated dynamic library loading, and enhances operational insight—all while preserving compatibility and ease of use. - - ## Goals **Introduce ToplingDB as a configurable and observable alternative to RocksDB.** Enable users to select ToplingDB via configuration, allowing tuning parameters through YAML files without recompilation and real-time monitoring via Web Server—without sacrificing compatibility with existing RocksDB APIs. - ## Design ### Configuration Parameters @@ -70,8 +63,6 @@ This parameter allows users to specify a YAML file that defines ToplingDB settin - **Fallback**: If the YAML file is not provided or ToplingDB is unavailable, HugeGraph will fall back to standard RocksDB behavior. - - #### `rocksdb.open_http`: Enable Web Server for Observability This boolean flag controls whether the embedded Web Server in ToplingDB should be started. The server exposes runtime metrics, configuration status, and internal RocksDB statistics via a browser-accessible interface. @@ -85,6 +76,7 @@ This boolean flag controls whether the embedded Web Server in ToplingDB should b ``` The listening port is defined in the YAML file specified by `option_path`, under the key `http.listening_ports`: + ```yaml http: document_root: /dev/shm/rocksdb_resource @@ -99,7 +91,6 @@ This boolean flag controls whether the embedded Web Server in ToplingDB should b - **Security**: The Web Server does **not** provide built-in authentication. In production environments, configure firewalls or network access controls carefully to prevent unauthorized access. - ### Reflection-Based Loading Mechanism To support ToplingDB without introducing hard dependencies, HugeGraph uses Java reflection to detect and load enhanced APIs at runtime. @@ -107,17 +98,15 @@ To support ToplingDB without introducing hard dependencies, HugeGraph uses Java During initialization, HugeGraph checks whether the current JAR contains the class `com.topling.sideplugin.SidePluginRepo`. If present, it assumes ToplingDB is available and proceeds to: 1. **Load the SidePluginRepo class via reflection** This avoids compile-time coupling and allows fallback to standard RocksDB if the class is missing. - * If the ToplingDB API cannot be found, HugeGraph silently falls back to the standard RocksDB API for startup. + - If the ToplingDB API cannot be found, HugeGraph silently falls back to the standard RocksDB API for startup. 2. **Invoke** `importAutoFile(optionPath)` This method parses the YAML configuration file specified by `rocksdb.option_path` to configure storage engine parameters. - * If the `option_path` is incorrect or parsing fails, ToplingDB throws an error and terminates the startup process. + - If the `option_path` is incorrect or parsing fails, ToplingDB throws an error and terminates the startup process. 3. **Call** `open()` **with a JSON descriptor** The parsed configuration is converted to a JSON structure and passed to the ToplingDB engine to initialize the database. 4. **Optionally start the Web Server** If `rocksdb.open_http` is true and the instance is `GRAPH_STORE`, HugeGraph invokes `startHttpServer()` via reflection to enable observability. - * If the Web Server cannot be started due to misconfiguration **or if the specified HTTP port is already in use**, ToplingDB throws an error and the startup process is terminated + - If the Web Server cannot be started due to misconfiguration **or if the specified HTTP port is already in use**, ToplingDB throws an error and the startup process is terminated This design ensures that ToplingDB can be integrated as an optional enhancement, without breaking compatibility or requiring changes to the core HugeGraph codebase. - - ## Impact ### For Users @@ -125,7 +114,6 @@ This design ensures that ToplingDB can be integrated as an optional enhancement, The way users operate remains unchanged by default, and adding ToplingDB configuration provides additional functionality. The ToplingDB integration is fully embedded into the existing startup scripts (`init-store.sh` and `start-hugegraph.sh`). Users only need to set `rocksdb.option_path` to specify the YAML file path and adjust its contents as needed to tune the storage engine. - ### For Developers Developers need to make two adjustments to enable ToplingDB during development: @@ -166,11 +154,11 @@ Developers need to make two adjustments to enable ToplingDB during development: The `preload-topling.sh` script not only extracts the necessary dynamic libraries and web server static resources into the `library` directory next to the `bin` directory, but also sets the required environment variables in the current process. When executed in a terminal using `source preload-topling.sh`, these variables take effect immediately in that shell session. - + However, when launching HugeGraph from an IDE, the program typically runs in a separate process, so environment variables defined in scripts run from the terminal are not inherited. In this case, developers need to manually configure the IDE's run/debug environment variables to ensure proper preloading of native libraries. - + In your IDE’s Run/Debug Configuration, set: ```bash @@ -180,11 +168,8 @@ Developers need to make two adjustments to enable ToplingDB during development: These steps ensure that ToplingDB loads correctly in development environments and behaves consistently with production deployments. - - ## Links -* **ToplingDB**: [https://github.com/topling/toplingdb](https://github.com/topling/toplingdb) -* **Configuration YAML of ToplingDB**: [https://github.com/topling/sideplugin-wiki-en/wiki](https://github.com/topling/sideplugin-wiki-en/wiki) -* **Web Server of ToplingDB**: [https://github.com/topling/sideplugin-wiki-en/wiki/WebView](https://github.com/topling/sideplugin-wiki-en/wiki/WebView) - +- **ToplingDB**: [https://github.com/topling/toplingdb](https://github.com/topling/toplingdb) +- **Configuration YAML of ToplingDB**: [https://github.com/topling/sideplugin-wiki-en/wiki](https://github.com/topling/sideplugin-wiki-en/wiki) +- **Web Server of ToplingDB**: [https://github.com/topling/sideplugin-wiki-en/wiki/WebView](https://github.com/topling/sideplugin-wiki-en/wiki/WebView) diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_plus.yaml b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_plus.yaml index d33d8da05a..a1a1601978 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_plus.yaml +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_plus.yaml @@ -18,7 +18,7 @@ http: # normally parent path of db path document_root: /dev/shm/rocksdb_resource - listening_ports: '2011' + listening_ports: '127.0.0.1:2011' setenv: StrSimpleEnvNameNotOverwrite: StringValue IntSimpleEnvNameNotOverwrite: 16384 diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java index 6f54e2f439..33816f5653 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java @@ -34,6 +34,16 @@ import org.rocksdb.DataBlockIndexType; import org.rocksdb.IndexType; +import java.util.regex.Pattern; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Locale; + +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.SafeConstructor; + import com.google.common.collect.ImmutableList; public class RocksDBOptions extends OptionHolder { @@ -44,6 +54,11 @@ private RocksDBOptions() { private static volatile RocksDBOptions instance; + private static final Pattern SAFE_PATH_PATTERN = + Pattern.compile("^[a-zA-Z0-9/_.-]+\\.yaml$"); + private static final String ALLOWED_CONFIG_DIR = "./conf/graphs/"; + private static final long MAX_CONFIG_FILE_SIZE = 1024 * 1024 * 10; // 10 MB + public static synchronized RocksDBOptions instance() { if (instance == null) { instance = new RocksDBOptions(); @@ -52,6 +67,68 @@ public static synchronized RocksDBOptions instance() { return instance; } + /** + * Validate the option_path string for safety and availability. + * - Enforce format, normalize and prevent path traversal + * - Restrict to an allowed base directory + * - Ensure the file exists, is readable, and within size limits + */ + public static void validateOptionPath(String optionPath) { + // 1. Path format validation + if (optionPath == null || optionPath.isBlank()) { + throw new IllegalArgumentException("option_path can't be null or empty"); + } + if (!SAFE_PATH_PATTERN.matcher(optionPath).matches() || + optionPath.contains("..") || optionPath.contains("://")) { + throw new IllegalArgumentException("Invalid option_path format: " + optionPath); + } + String lower = optionPath.toLowerCase(Locale.ROOT); + if (!(lower.endsWith(".yaml") || lower.endsWith(".yml"))) { + throw new IllegalArgumentException("option_path must end with .yaml or .yml"); + } + + // 2. Normalize path and constrain under allowed directory + Path allowedDir = Paths.get(ALLOWED_CONFIG_DIR).toAbsolutePath().normalize(); + Path configPath = Paths.get(optionPath).toAbsolutePath().normalize(); + + if (!configPath.startsWith(allowedDir)) { + throw new SecurityException("option_path must be under " + ALLOWED_CONFIG_DIR); + } + + // 3. Validate file existence and readability + if (!Files.isRegularFile(configPath) || !Files.isReadable(configPath)) { + throw new IllegalArgumentException( + "Config file not found or not readable: " + configPath); + } + + // 4. File size limit (prevent DoS) + final long fileSize; + try { + fileSize = Files.size(configPath); + } catch (IOException e) { + throw new IllegalArgumentException("Failed to access config file size: " + configPath, + e); + } + if (fileSize > MAX_CONFIG_FILE_SIZE) { + throw new IllegalArgumentException("Config file too large: " + fileSize + " bytes"); + } + } + + /** + * - Validates parsed structure schema and limits depth & node counts + */ + public static void validateYamlContent(String yamlContent) { + // Use a safe YAML parser and disable dangerous features + Yaml yaml = new Yaml(new SafeConstructor()); + try { + yaml.load(yamlContent); + // TODO: validate config schema + } catch (Exception e) { + throw new IllegalArgumentException( + "Invalid YAML configuration", e); + } + } + // TODO: the entire align style is wrong, change it to 4 space later public static final ConfigOption DATA_PATH = new ConfigOption<>( diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java index 81bc01c437..9853eab7e4 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java @@ -447,6 +447,8 @@ private static OpenedRocksDB openRocksDB(HugeConfig config, String dataPath, boolean useTopling = false; if (!StringUtils.isEmpty(optionPath)) { try { + RocksDBOptions.validateOptionPath(optionPath); + RocksDBOptions.validateYamlContent(optionPath); Class.forName("org.rocksdb.SidePluginRepo"); useTopling = true; LOG.info("SidePluginRepo found. Will attempt to open default CF RocksDB using " + @@ -552,6 +554,8 @@ private static OpenedRocksDB openRocksDB(HugeConfig config, boolean useTopling = false; if (!StringUtils.isEmpty(optionPath)) { try { + RocksDBOptions.validateOptionPath(optionPath); + RocksDBOptions.validateYamlContent(optionPath); Class.forName("org.rocksdb.SidePluginRepo"); useTopling = true; LOG.info("SidePluginRepo found. Will attempt to open multi CFs RocksDB using " + From 63f607f4466eb81174dcf55ad66565d48b912d68 Mon Sep 17 00:00:00 2001 From: gydeng Date: Wed, 15 Oct 2025 06:39:20 +0000 Subject: [PATCH 34/51] fix: update ci scripts --- .github/workflows/pd-store-ci.yml | 3 +- .github/workflows/server-ci.yml | 3 +- docs/toplingdb/toplingdb.md | 6 +- .../src/assembly/static/bin/common-topling.sh | 77 ++++++++++++++++--- .../src/assembly/travis/install-deps.sh | 24 ++++++ 5 files changed, 99 insertions(+), 14 deletions(-) create mode 100755 hugegraph-server/hugegraph-dist/src/assembly/travis/install-deps.sh diff --git a/.github/workflows/pd-store-ci.yml b/.github/workflows/pd-store-ci.yml index ca4eff81e3..99b5879ae2 100644 --- a/.github/workflows/pd-store-ci.yml +++ b/.github/workflows/pd-store-ci.yml @@ -184,8 +184,7 @@ jobs: - name: Install deps run: | - sudo apt-get update -y - sudo apt-get install -y liburing-dev libaio-dev libjemalloc-dev + ${TRAVIS_DIR}/install-deps.sh - name: use staged maven repo settings if: ${{ env.USE_STAGE == 'true' }} diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index 48f59b7d01..affb684384 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -56,8 +56,7 @@ jobs: - name: Install deps run: | - sudo apt-get update -y - sudo apt-get install -y liburing-dev libaio-dev libjemalloc-dev + ${TRAVIS_DIR}/install-deps.sh - name: Cache Maven packages uses: actions/cache@v4 diff --git a/docs/toplingdb/toplingdb.md b/docs/toplingdb/toplingdb.md index 09bdcf787d..2a0566fde0 100644 --- a/docs/toplingdb/toplingdb.md +++ b/docs/toplingdb/toplingdb.md @@ -57,6 +57,8 @@ This parameter allows users to specify a YAML file that defines ToplingDB settin The specified YAML file will be automatically loaded during database initialization if ToplingDB is available. + For security reasons, HugeGraph only allows YAML files to be stored under the `$HUGEGRAPH_HOME/conf/graphs` directory. + For details on the YAML structure and supported configuration fields, please refer to [SidePlugin](https://github.com/topling/sideplugin-wiki-en/wiki). - **Implementation**: During initialization, HugeGraph checks whether the configured JAR contains ToplingDB APIs. If so, it uses reflection to load the SidePluginRepo class and calls `importAutoFile(optionPath)` to parse the YAML file. The resulting configuration is applied to the RocksDB instance. @@ -80,9 +82,11 @@ This boolean flag controls whether the embedded Web Server in ToplingDB should b ```yaml http: document_root: /dev/shm/rocksdb_resource - listening_ports: '2011' + listening_ports: '127.0.0.1:2011' # by default, only local access is allowed ``` + For security reasons, the default configuration only allows local access. + When adjusting this setting, users should carefully manage port and network access permissions to avoid potential security incidents. To preview the Web Server interface and its layout, see [Web Server](https://github.com/topling/sideplugin-wiki-en/wiki/WebView). - **Implementation**: If `open_http` is set to true and the database instance is `GRAPH_STORE`, HugeGraph invokes `startHttpServer()` on the ToplingDB repo object. This exposes a browser-accessible dashboard for monitoring RocksDB internals. diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh index 7a1353c831..f264eb2a0a 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh @@ -19,6 +19,10 @@ set -Eeuo pipefail IFS=$'\n\t' trap 'echo "[common-topling] error at line ${LINENO}: ${BASH_COMMAND}" >&2' ERR +BIN="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TOP="$(cd "$BIN"/../ && pwd)" +GITHUB="https://github.com" + function abs_path() { local SOURCE SOURCE="${BASH_SOURCE[0]}" @@ -144,6 +148,67 @@ function ensure_libaio_symlink() { fi } +function download_and_verify() { + local url=$1 + local filepath=$2 + local expected_md5=$3 + + if [[ -f $filepath ]]; then + echo "File $filepath exists. Verifying MD5 checksum..." + actual_md5=$(md5sum $filepath | awk '{ print $1 }') + if [[ $actual_md5 != $expected_md5 ]]; then + echo "MD5 checksum verification failed for $filepath. Expected: $expected_md5, but got: $actual_md5" + echo "Deleting $filepath..." + rm -f $filepath + else + echo "MD5 checksum verification succeeded for $filepath." + return 0 + fi + fi + + echo "Downloading $filepath..." + curl -L -o $filepath $url + + actual_md5=$(md5sum $filepath | awk '{ print $1 }') + if [[ $actual_md5 != $expected_md5 ]]; then + echo "MD5 checksum verification failed for $filepath after download. Expected: $expected_md5, but got: $actual_md5" + return 1 + fi + + return 0 +} + +function download_and_setup_jemalloc() { + local arch lib_file download_url expected_md5 + + # Detect system architecture + arch=$(uname -m) + + # System jemalloc not found, try to download the correct library for the architecture + if [[ $arch == "aarch64" || $arch == "arm64" ]]; then + lib_file="$TOP/bin/libjemalloc_aarch64.so" + download_url="${GITHUB}/apache/hugegraph-doc/raw/binary-1.5/dist/server/libjemalloc_aarch64.so" + expected_md5="2a631d2f81837f9d5864586761c5e380" + elif [[ $arch == "x86_64" ]]; then + lib_file="$TOP/bin/libjemalloc.so" + download_url="${GITHUB}/apache/hugegraph-doc/raw/binary-1.5/dist/server/libjemalloc.so" + expected_md5="fd61765eec3bfea961b646c269f298df" + else + echo "Unsupported architecture: $arch" + return 1 + fi + + # Download and verify jemalloc library + if download_and_verify "$download_url" "$lib_file" "$expected_md5"; then + if [[ ":${LD_PRELOAD:-}:" != *"libjemalloc.so:"* ]]; then + export LD_PRELOAD="${lib_file}${LD_PRELOAD:+:$LD_PRELOAD}" + fi + else + echo "Failed to verify or download jemalloc for $arch, skipping" + return 1 + fi +} + function preload_toplingdb() { local lib_dir="$1" local dest_dir="$2" @@ -156,24 +221,18 @@ function preload_toplingdb() { fi ensure_libaio_symlink + download_and_setup_jemalloc extract_so_with_jar "$jar_file" "$dest_dir" if [ -d "$dest_dir" ]; then if [[ ":${LD_LIBRARY_PATH:-}:" != *":$dest_dir:"* ]]; then export LD_LIBRARY_PATH="$dest_dir${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" fi - if [ -f "$dest_dir/librocksdbjni-linux64.so" ] && [[ ":${LD_PRELOAD:-}:" != *":librocksdbjni-linux64.so:"* ]]; then - export LD_PRELOAD="librocksdbjni-linux64.so${LD_PRELOAD:+:$LD_PRELOAD}" + if [ -f "$dest_dir/librocksdbjni-linux64.so" ] && [[ ":${LD_PRELOAD:-}:" != *"librocksdbjni-linux64.so:"* ]]; then + export LD_PRELOAD="${LD_PRELOAD:+$LD_PRELOAD:}$dest_dir/librocksdbjni-linux64.so" fi else echo "Warn: LD paths skipped, directory '$dest_dir' does not exist." >&2 fi - if command -v ldconfig >/dev/null 2>&1; then - local jemalloc_found - jemalloc_found=$(ldconfig -p 2>/dev/null | grep -F 'libjemalloc.so' || true) - if [ -n "$jemalloc_found" ] && [[ ":${LD_PRELOAD:-}:" != *":libjemalloc.so:"* ]]; then - export LD_PRELOAD="libjemalloc.so${LD_PRELOAD:+:$LD_PRELOAD}" - fi - fi extract_html_css_from_jar "$jar_file" "$dest_dir" } diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/install-deps.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-deps.sh new file mode 100755 index 0000000000..c5719254a4 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-deps.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -ev + +if [ -f /etc/debian_version ]; then + sudo apt-get update && sudo apt-get install -y liburing-dev libaio-dev libjemalloc-dev +elif [ -f /etc/redhat-release ]; then + sudo yum install -y liburing-devel libaio-devel jemalloc-devel +fi From f8023c4de0e81defacdbb94f72caf5b998d99ad6 Mon Sep 17 00:00:00 2001 From: gydeng Date: Tue, 4 Nov 2025 08:35:21 +0000 Subject: [PATCH 35/51] fix(server): fix environment issues in the script --- .../src/assembly/static/bin/common-topling.sh | 52 +++++++++++++------ .../assembly/static/bin/preload-topling.sh | 28 ++++++---- .../static/conf/graphs/hugegraph.properties | 2 +- ...{rocksdb_plus.yaml => rocksdb_server.yaml} | 2 +- 4 files changed, 55 insertions(+), 29 deletions(-) rename hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/{rocksdb_plus.yaml => rocksdb_server.yaml} (99%) diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh index f264eb2a0a..c69b6e0c6e 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh @@ -74,7 +74,6 @@ function extract_html_css_from_jar() { local jar_file="$1" local dest_dir="$2" local abs_jar_path - local resource_target="/dev/shm/rocksdb_resource" if [ ! -f "$jar_file" ]; then echo "Error: JAR file '$jar_file' does not exist." >&2 @@ -105,18 +104,6 @@ function extract_html_css_from_jar() { return $code fi } - - mkdir -p "$resource_target" || { - echo "Error: Cannot create target directory '$resource_target'." >&2 - return 1 - } - - if compgen -G "$dest_dir"/*.html >/dev/null 2>&1; then - cp -f "$dest_dir"/*.html "$resource_target"/ - fi - if compgen -G "$dest_dir"/*.css >/dev/null 2>&1; then - cp -f "$dest_dir"/*.css "$resource_target"/ - fi } function ensure_libaio_symlink() { @@ -141,8 +128,6 @@ function ensure_libaio_symlink() { else echo "Warn: libaio.so.1t64 not found, skip creating compat symlink" >&2 fi - else - echo "libaio.so.1 found, skip creating compatibility symlink" >&2 fi fi fi @@ -179,7 +164,40 @@ function download_and_verify() { } function download_and_setup_jemalloc() { - local arch lib_file download_url expected_md5 + local arch lib_file download_url expected_md5 system_lib + + # Prefer system-installed jemalloc if available + # Try ldconfig first to locate the shared object + if command -v ldconfig >/dev/null 2>&1; then + system_lib=$(ldconfig -p 2>/dev/null | awk '/jemalloc/{print $4}' | head -n1) + fi + # Fallback to common library paths if ldconfig is not available or found nothing + if [[ -z "$system_lib" ]]; then + for p in \ + /usr/lib/libjemalloc.so \ + /usr/lib/libjemalloc.so.2 \ + /usr/lib64/libjemalloc.so \ + /usr/lib64/libjemalloc.so.2 \ + /usr/local/lib/libjemalloc.so \ + /usr/local/lib/libjemalloc.so.2 \ + /usr/lib/x86_64-linux-gnu/libjemalloc.so \ + /usr/lib/x86_64-linux-gnu/libjemalloc.so.2 \ + /usr/lib/aarch64-linux-gnu/libjemalloc.so \ + /usr/lib/aarch64-linux-gnu/libjemalloc.so.2; do + if [[ -f "$p" ]]; then + system_lib="$p" + break + fi + done + fi + + # If found, set LD_PRELOAD and return immediately + if [[ -n "$system_lib" ]]; then + if [[ ":${LD_PRELOAD:-}:" != *"libjemalloc"* ]]; then + export LD_PRELOAD="${system_lib}${LD_PRELOAD:+:$LD_PRELOAD}" + fi + return 0 + fi # Detect system architecture arch=$(uname -m) @@ -198,7 +216,7 @@ function download_and_setup_jemalloc() { return 1 fi - # Download and verify jemalloc library + # Download and verify jemalloc library (fallback when system lib not found) if download_and_verify "$download_url" "$lib_file" "$expected_md5"; then if [[ ":${LD_PRELOAD:-}:" != *"libjemalloc.so:"* ]]; then export LD_PRELOAD="${lib_file}${LD_PRELOAD:+:$LD_PRELOAD}" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh index 1caced578d..734bc48e8b 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh @@ -17,24 +17,32 @@ # set -Eeuo pipefail +# Save original IFS to avoid leaking into parent shell when sourced +ORIG_IFS="${IFS}" IFS=$'\n\t' # Unified error capture for easy positioning trap 'echo "[preload-topling] error at line ${LINENO}: ${BASH_COMMAND}" >&2' ERR -BIN="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -TOP="$(cd "$BIN"/../ && pwd)" -LIB="$TOP/lib" -DEST_DIR="$TOP/library" +SERVER_BIN="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SERVER_TOP="$(cd "$SERVER_BIN"/../ && pwd)" +SERVER_LIB="$SERVER_TOP/lib" +DEST_DIR="$SERVER_TOP/library" -if [ ! -d "$LIB" ]; then - echo "Error: LIB dir not found: $LIB" >&2 +if [ ! -d "$SERVER_LIB" ]; then + echo "Error: LIB dir not found: $SERVER_LIB" >&2 exit 1 fi -if [ ! -f "$BIN/common-topling.sh" ]; then - echo "Error: common-topling.sh not found under: $BIN" >&2 +if [ ! -f "$SERVER_BIN/common-topling.sh" ]; then + echo "Error: common-topling.sh not found under: $SERVER_BIN" >&2 exit 1 fi -source "$BIN/common-topling.sh" +source "$SERVER_BIN/common-topling.sh" type preload_toplingdb >/dev/null 2>&1 || { echo "Error: function preload_toplingdb not found" >&2; exit 1; } -preload_toplingdb "$LIB" "$DEST_DIR" +preload_toplingdb "$SERVER_LIB" "$DEST_DIR" + +# Reset shell options to prevent affecting the parent shell when sourced +set +Eeuo pipefail +trap - ERR +# Restore original IFS +IFS="$ORIG_IFS" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/hugegraph.properties b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/hugegraph.properties index fe17100b91..8262d0b2da 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/hugegraph.properties +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/hugegraph.properties @@ -42,7 +42,7 @@ search.text_analyzer_mode=INDEX # rocksdb backend config #rocksdb.data_path=/path/to/disk #rocksdb.wal_path=/path/to/disk -#rocksdb.option_path=./conf/graphs/rocksdb_plus.yaml +#rocksdb.option_path=./conf/graphs/rocksdb_server.yaml #rocksdb.open_http=true # hbase backend config diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_plus.yaml b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_server.yaml similarity index 99% rename from hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_plus.yaml rename to hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_server.yaml index a1a1601978..440c515056 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_plus.yaml +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_server.yaml @@ -17,7 +17,7 @@ # common parameters http: # normally parent path of db path - document_root: /dev/shm/rocksdb_resource + document_root: ./library listening_ports: '127.0.0.1:2011' setenv: StrSimpleEnvNameNotOverwrite: StringValue From 46fb13c847b563226cdd7cf53661a01c2b6d85d6 Mon Sep 17 00:00:00 2001 From: gydeng Date: Tue, 4 Nov 2025 08:48:41 +0000 Subject: [PATCH 36/51] feat(rocksdb-provider): refactor the code to be compatible with toplingdb --- hugegraph-rocksdb-provider/pom.xml | 89 ++++ .../provider/AbstractRocksDBProvider.java | 198 +++++++ .../rocksdb/provider/RocksDBProvider.java | 164 ++++++ .../provider/RocksDBProviderLoader.java | 276 ++++++++++ .../provider/StandardRocksDBProvider.java | 146 ++++++ .../provider/ToplingRocksDBProvider.java | 490 ++++++++++++++++++ ...hugegraph.rocksdb.provider.RocksDBProvider | 2 + pom.xml | 1 + 8 files changed, 1366 insertions(+) create mode 100644 hugegraph-rocksdb-provider/pom.xml create mode 100644 hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/AbstractRocksDBProvider.java create mode 100644 hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/RocksDBProvider.java create mode 100644 hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/RocksDBProviderLoader.java create mode 100644 hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/StandardRocksDBProvider.java create mode 100644 hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/ToplingRocksDBProvider.java create mode 100644 hugegraph-rocksdb-provider/src/main/resources/META-INF/services/org.apache.hugegraph.rocksdb.provider.RocksDBProvider diff --git a/hugegraph-rocksdb-provider/pom.xml b/hugegraph-rocksdb-provider/pom.xml new file mode 100644 index 0000000000..fc7be341cf --- /dev/null +++ b/hugegraph-rocksdb-provider/pom.xml @@ -0,0 +1,89 @@ + + + + + hugegraph + org.apache.hugegraph + ${revision} + ../pom.xml + + 4.0.0 + + hugegraph-rocksdb-provider + ${project.artifactId} + + HugeGraph RocksDB Provider - A common adapter module for RocksDB and ToplingDB integration + using SPI (Service Provider Interface) and Strategy Pattern for loose coupling and plugin-based architecture. + + + + + + org.rocksdb + rocksdbjni + 8.10.2-SNAPSHOT + + + + + org.apache.hugegraph + hugegraph-common + ${hugegraph-commons.version} + + + + + org.slf4j + slf4j-api + 1.7.5 + + + + + net.minidev + json-smart + 2.4.8 + + + + + junit + junit + 4.13.1 + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 11 + 11 + UTF-8 + + + + + + diff --git a/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/AbstractRocksDBProvider.java b/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/AbstractRocksDBProvider.java new file mode 100644 index 0000000000..515d3e66f6 --- /dev/null +++ b/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/AbstractRocksDBProvider.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.rocksdb.provider; + +import org.rocksdb.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.List; + +/** + * Abstract base class for RocksDB providers that provides common utility methods. + * This class focuses only on the core abstraction of RocksDB open/close operations + * without complex configuration management. + */ +public abstract class AbstractRocksDBProvider implements RocksDBProvider { + + private static final Logger LOG = LoggerFactory.getLogger(AbstractRocksDBProvider.class); + + @Override + public final RocksDB openRocksDB(Options options, String dataPath) throws RocksDBException { + LOG.debug("Opening RocksDB with provider: {} at path: {}", getProviderName(), dataPath); + + // Ensure directory exists + ensureDirectoryExists(dataPath); + + // Initialize provider if needed + initialize(); + + try { + // Delegate to provider-specific implementation + RocksDB rocksDB = doOpenRocksDB(options, dataPath); + + return rocksDB; + } catch (Exception e) { + LOG.error("Failed to open RocksDB with provider: {} at path: {}", + getProviderName(), dataPath, e); + throw e; + } + } + + @Override + public final RocksDB openRocksDB(DBOptions dbOptions, String dataPath, + List cfDescriptors, + List cfHandles) throws RocksDBException { + // Ensure directory exists + ensureDirectoryExists(dataPath); + + // Initialize provider if needed + initialize(); + + try { + // Delegate to provider-specific implementation + RocksDB rocksDB = doOpenRocksDB(dbOptions, dataPath, cfDescriptors, cfHandles); + + return rocksDB; + } catch (Exception e) { + LOG.error("Failed to open RocksDB with column families using provider: {} at path: {}", + getProviderName(), dataPath, e); + throw e; + } + } + + @Override + public final RocksDB openRocksDB(DBOptions dbOptions, String dataPath, + List cfDescriptors, + List cfHandles, + String optionPath, Boolean openHttp) throws RocksDBException { + + LOG.debug("Opening RocksDB with extended parameters using provider: {} at path: {}", + getProviderName(), dataPath); + + // Ensure directory exists + ensureDirectoryExists(dataPath); + + // Initialize provider if needed + initialize(); + + try { + // Delegate to provider-specific implementation + RocksDB rocksDB = + doOpenRocksDB(dbOptions, dataPath, cfDescriptors, cfHandles, optionPath, + openHttp); + return rocksDB; + } catch (Exception e) { + throw e; + } + } + + @Override + public void closeRocksDB(RocksDB rocksDB) { + if (rocksDB != null) { + // Perform provider-specific close operations + performProviderSpecificClose(rocksDB); + rocksDB.close(); + } + } + + /** + * Ensure database directory exists + */ + protected void ensureDirectoryExists(String dbPath) { + File dbDir = new File(dbPath); + if (!dbDir.exists()) { + boolean created = dbDir.mkdirs(); + if (!created) { + throw new RuntimeException("Failed to create database directory: " + dbPath); + } + LOG.info("Created database directory: {}", dbPath); + } + } + + // ========== Abstract methods for provider-specific implementations ========== + + /** + * Provider-specific implementation for opening RocksDB with Options + * + * @param options RocksDB options + * @param dataPath database path + * @return opened RocksDB instance + * @throws RocksDBException if opening fails + */ + protected abstract RocksDB doOpenRocksDB(Options options, String dataPath) + throws RocksDBException; + + /** + * Provider-specific implementation for opening RocksDB with Options and extended parameters + * + * @param options RocksDB options + * @param dataPath database path + * @param optionPath optional configuration file path (can be null) + * @param openHttp whether to start HTTP server (can be null, defaults to false) + * @return opened RocksDB instance + * @throws RocksDBException if opening fails + */ + protected abstract RocksDB doOpenRocksDB(Options options, String dataPath, String optionPath, + Boolean openHttp) throws RocksDBException; + + /** + * Provider-specific implementation for opening RocksDB with column families + * + * @param dbOptions database options + * @param dataPath database path + * @param cfDescriptors column family descriptors + * @param cfHandles list to store column family handles + * @return opened RocksDB instance + * @throws RocksDBException if opening fails + */ + protected abstract RocksDB doOpenRocksDB(DBOptions dbOptions, String dataPath, + List cfDescriptors, + List cfHandles) + throws RocksDBException; + + /** + * Provider-specific implementation for opening RocksDB with extended parameters + * + * @param dbOptions database options + * @param dataPath database path + * @param cfDescriptors column family descriptors + * @param cfHandles list to store column family handles + * @param optionPath optional configuration file path (can be null) + * @param openHttp whether to start HTTP server (can be null, defaults to false) + * @return opened RocksDB instance + * @throws RocksDBException if opening fails + */ + protected abstract RocksDB doOpenRocksDB(DBOptions dbOptions, String dataPath, + List cfDescriptors, + List cfHandles, + String optionPath, Boolean openHttp) + throws RocksDBException; + + /** + * Template method for provider-specific close operations. + * Subclasses can override this method to perform additional cleanup. + * + * @param rocksDB RocksDB instance being closed + */ + protected void performProviderSpecificClose(RocksDB rocksDB) { + // Default implementation does nothing + // Subclasses can override for specific cleanup + } +} diff --git a/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/RocksDBProvider.java b/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/RocksDBProvider.java new file mode 100644 index 0000000000..8e78b4ff34 --- /dev/null +++ b/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/RocksDBProvider.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.rocksdb.provider; + +import org.rocksdb.*; + +import java.util.List; + +/** + * Simplified RocksDB Provider SPI interface for pluggable RocksDB implementations. + * This interface only abstracts the core RocksDB.open() and rocksdb.close() operations, + * making it a direct replacement for standard RocksDB API calls. + *

+ * The design philosophy is to keep it minimal - only replace the open/close operations + * while maintaining full compatibility with standard RocksDB API signatures. + *

+ * Implementations should be registered via Java SPI mechanism in + * META-INF/services/org.apache.hugegraph.rocksdb.provider.RocksDBProvider + */ +public interface RocksDBProvider { + + /** + * Get the provider name/type identifier + * + * @return provider name (e.g., "standard", "topling") + */ + String getProviderName(); + + /** + * Get the priority of this provider (higher priority providers are preferred) + * + * @return priority value + */ + int getPriority(); + + /** + * Check if this provider is available in the current environment + * + * @return true if the provider can be used + */ + boolean isAvailable(); + + // ========== Core RocksDB Open Operations ========== + + /** + * Open RocksDB - direct replacement for RocksDB.open(options, dataPath) + * + * @param options RocksDB options + * @param dataPath database path + * @return opened RocksDB instance + * @throws RocksDBException if opening fails + */ + RocksDB openRocksDB(Options options, String dataPath) throws RocksDBException; + + /** + * Open RocksDB with additional parameters for special providers (like ToplingDB) + * + * @param options RocksDB options + * @param dataPath database path + * @param optionPath optional configuration file path (can be null) + * @param openHttp whether to start HTTP server (can be null, defaults to false) + * @return opened RocksDB instance + * @throws RocksDBException if opening fails + */ + RocksDB openRocksDB(Options options, String dataPath, String optionPath, Boolean openHttp) + throws RocksDBException; + + /** + * Open RocksDB with column families - direct replacement for + * RocksDB.open(dbOptions, dataPath, cfDescriptors, cfHandles) + * + * @param dbOptions database options + * @param dataPath database path + * @param cfDescriptors column family descriptors + * @param cfHandles list to store column family handles + * @return opened RocksDB instance + * @throws RocksDBException if opening fails + */ + RocksDB openRocksDB(DBOptions dbOptions, String dataPath, + List cfDescriptors, + List cfHandles) throws RocksDBException; + + /** + * Open RocksDB with additional parameters for special providers (like ToplingDB) + * This method supports optionPath and openHttp parameters while maintaining + * the same signature as the standard openRocksDB method. + * + * @param dbOptions database options + * @param dataPath database path + * @param cfDescriptors column family descriptors + * @param cfHandles list to store column family handles + * @param optionPath optional configuration file path (can be null) + * @param openHttp whether to start HTTP server (can be null, defaults to false) + * @return opened RocksDB instance + * @throws RocksDBException if opening fails + */ + RocksDB openRocksDB(DBOptions dbOptions, String dataPath, + List cfDescriptors, + List cfHandles, + String optionPath, Boolean openHttp) throws RocksDBException; + + // ========== Core RocksDB Close Operations ========== + + /** + * Close RocksDB - direct replacement for rocksdb.close() + * This method handles both the RocksDB instance and any associated column family handles. + * + * @param rocksDB RocksDB instance to close + */ + void closeRocksDB(RocksDB rocksDB); + + /** + * Close RocksDB with column family handles + * + * @param rocksDB RocksDB instance to close + * @param cfHandles column family handles to close (can be null) + */ + default void closeRocksDB(RocksDB rocksDB, List cfHandles) { + // Close column family handles first + if (cfHandles != null) { + for (ColumnFamilyHandle cfHandle : cfHandles) { + if (cfHandle != null) { + cfHandle.close(); + } + } + } + + // Close the RocksDB instance + closeRocksDB(rocksDB); + } + + // ========== Provider Lifecycle ========== + + /** + * Perform provider-specific initialization + * This method is called once when the provider is first used. + */ + default void initialize() { + // Default implementation does nothing + } + + /** + * Perform provider-specific cleanup + * This method is called when the provider is no longer needed. + */ + default void shutdown() { + // Default implementation does nothing + } +} diff --git a/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/RocksDBProviderLoader.java b/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/RocksDBProviderLoader.java new file mode 100644 index 0000000000..d95bccc676 --- /dev/null +++ b/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/RocksDBProviderLoader.java @@ -0,0 +1,276 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.rocksdb.provider; + +import org.rocksdb.ColumnFamilyDescriptor; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.DBOptions; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +/** + * RocksDB Provider SPI Loader that manages the loading and selection + * of RocksDB providers using Java's ServiceLoader mechanism. + */ +public class RocksDBProviderLoader { + + private static final Logger LOG = LoggerFactory.getLogger(RocksDBProviderLoader.class); + + private static final RocksDBProviderLoader INSTANCE = new RocksDBProviderLoader(); + + private final Map providerCache = new ConcurrentHashMap<>(); + private volatile boolean loaded = false; + + private RocksDBProviderLoader() { + // Private constructor for singleton + } + + public static RocksDBProviderLoader getInstance() { + return INSTANCE; + } + + /** + * Load all available RocksDB providers using SPI + */ + public synchronized void loadProviders() { + if (loaded) { + return; + } + + LOG.info("Loading RocksDB providers via SPI..."); + + ServiceLoader serviceLoader = ServiceLoader.load(RocksDBProvider.class); + + for (RocksDBProvider provider : serviceLoader) { + try { + if (provider.isAvailable()) { + providerCache.put(provider.getProviderName(), provider); + LOG.info("Loaded RocksDB provider: {} (priority: {})", + provider.getProviderName(), provider.getPriority()); + } else { + LOG.warn("RocksDB provider {} is not available in current environment", + provider.getProviderName()); + } + } catch (Exception e) { + LOG.error("Failed to load RocksDB provider: {}", provider.getClass().getName(), e); + } + } + + if (providerCache.isEmpty()) { + LOG.warn( + "No RocksDB providers found! Make sure providers are properly registered in " + + "META-INF/services"); + } else { + LOG.info("Successfully loaded {} RocksDB provider(s): {}", + providerCache.size(), providerCache.keySet()); + } + + loaded = true; + } + + /** + * Get a specific provider by name + * + * @param providerName provider name + * @return RocksDB provider or null if not found + */ + public RocksDBProvider getProvider(String providerName) { + if (!loaded) { + loadProviders(); + } + + return providerCache.get(providerName); + } + + /** + * Get the best available provider based on priority + * + * @return best available RocksDB provider + */ + public RocksDBProvider getBestProvider() { + if (!loaded) { + loadProviders(); + } + + if (providerCache.isEmpty()) { + throw new RuntimeException("No RocksDB providers available"); + } + + // Find provider with highest priority + RocksDBProvider bestProvider = null; + int highestPriority = Integer.MIN_VALUE; + + for (RocksDBProvider provider : providerCache.values()) { + if (provider.isAvailable() && provider.getPriority() > highestPriority) { + bestProvider = provider; + highestPriority = provider.getPriority(); + } + } + + if (bestProvider == null) { + throw new RuntimeException("No available RocksDB providers found"); + } + + LOG.info("Auto-selected RocksDB provider: {} (priority: {})", + bestProvider.getProviderName(), bestProvider.getPriority()); + return bestProvider; + } + + /** + * Get all loaded providers + * + * @return collection of all providers + */ + public Collection getAllProviders() { + if (!loaded) { + loadProviders(); + } + + return Collections.unmodifiableCollection(providerCache.values()); + } + + /** + * Get names of all available providers + * + * @return set of provider names + */ + public Set getAvailableProviderNames() { + if (!loaded) { + loadProviders(); + } + + return Collections.unmodifiableSet(providerCache.keySet()); + } + + /** + * Check if a specific provider is available + * + * @param providerName provider name + * @return true if provider is available + */ + public boolean isProviderAvailable(String providerName) { + if (!loaded) { + loadProviders(); + } + + RocksDBProvider provider = providerCache.get(providerName); + return provider != null && provider.isAvailable(); + } + + /** + * Reload all providers + */ + public synchronized void reload() { + loaded = false; + providerCache.clear(); + loadProviders(); + } + + // Static convenience methods + + /** + * Open RocksDB with simple options + * + * @param options RocksDB options + * @param dataPath database path + * @return opened RocksDB instance + * @throws RocksDBException if opening fails + */ + public static RocksDB openRocksDB(Options options, String dataPath) throws RocksDBException { + return openRocksDB(options, dataPath, null, null); + } + + /** + * Open RocksDB with options and optional parameters + * + * @param options RocksDB options + * @param dataPath database path + * @param optionPath optional path to options file + * @param openHttp optional HTTP server flag + * @return opened RocksDB instance + * @throws RocksDBException if opening fails + */ + public static RocksDB openRocksDB(Options options, String dataPath, String optionPath, + Boolean openHttp) throws RocksDBException { + RocksDBProvider provider = getInstance().getBestProvider(); + return provider.openRocksDB(options, dataPath, optionPath, openHttp); + } + + /** + * Open RocksDB with column families + * + * @param dbOptions database options + * @param dataPath database path + * @param cfDescriptors column family descriptors + * @param cfHandles column family handles (output) + * @return opened RocksDB instance + * @throws RocksDBException if opening fails + */ + public static RocksDB openRocksDB(DBOptions dbOptions, String dataPath, + List cfDescriptors, + List cfHandles) throws RocksDBException { + return openRocksDB(dbOptions, dataPath, cfDescriptors, cfHandles, null, null); + } + + /** + * Open RocksDB with column families and optional parameters + * + * @param dbOptions database options + * @param dataPath database path + * @param cfDescriptors column family descriptors + * @param cfHandles column family handles (output) + * @param optionPath optional path to options file + * @param openHttp optional HTTP server flag + * @return opened RocksDB instance + * @throws RocksDBException if opening fails + */ + public static RocksDB openRocksDB(DBOptions dbOptions, String dataPath, + List cfDescriptors, + List cfHandles, + String optionPath, Boolean openHttp) throws RocksDBException { + RocksDBProvider provider = getInstance().getBestProvider(); + return provider.openRocksDB(dbOptions, dataPath, cfDescriptors, cfHandles, optionPath, + openHttp); + } + + /** + * Close RocksDB instance + * + * @param rocksDB RocksDB instance to close + */ + public static void closeRocksDB(RocksDB rocksDB) { + RocksDBProvider provider = getInstance().getBestProvider(); + provider.closeRocksDB(rocksDB); + } + + /** + * Get provider by name (static method) + * + * @param providerName provider name + * @return RocksDB provider or null if not found + */ + public static RocksDBProvider getProviderByName(String providerName) { + return getInstance().getProvider(providerName); + } +} diff --git a/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/StandardRocksDBProvider.java b/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/StandardRocksDBProvider.java new file mode 100644 index 0000000000..ef571c5eda --- /dev/null +++ b/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/StandardRocksDBProvider.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.rocksdb.provider; + +import org.rocksdb.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +/** + * Standard RocksDB provider implementation that uses the standard RocksDB library. + * This provider handles the traditional RocksDB opening and configuration logic. + */ +public class StandardRocksDBProvider extends AbstractRocksDBProvider { + + private static final Logger LOG = LoggerFactory.getLogger(StandardRocksDBProvider.class); + + private static final String PROVIDER_NAME = "standard"; + private static final int PROVIDER_PRIORITY = 100; // Lower priority than ToplingDB + + @Override + public String getProviderName() { + return PROVIDER_NAME; + } + + @Override + public int getPriority() { + return PROVIDER_PRIORITY; + } + + @Override + public boolean isAvailable() { + try { + // Check if standard RocksDB is available + RocksDB.loadLibrary(); + return true; + } catch (Exception e) { + LOG.warn("Standard RocksDB is not available: {}", e.getMessage()); + return false; + } + } + + @Override + protected RocksDB doOpenRocksDB(Options options, String dataPath) throws RocksDBException { + try { + return RocksDB.open(options, dataPath); + } catch (RocksDBException e) { + throw e; + } + } + + @Override + public RocksDB openRocksDB(Options options, String dataPath, String optionPath, + Boolean openHttp) throws RocksDBException { + return doOpenRocksDB(options, dataPath, optionPath, openHttp); + } + + @Override + protected RocksDB doOpenRocksDB(Options options, String dataPath, String optionPath, + Boolean openHttp) throws RocksDBException { + LOG.debug("Opening standard RocksDB with Options and extended parameters at path: {}", + dataPath); + + // Log warnings for unsupported parameters + if (optionPath != null) { + LOG.warn("Standard RocksDB does not support optionPath parameter, ignoring: {}", + optionPath); + } + if (openHttp != null && openHttp) { + LOG.warn("Standard RocksDB does not support HTTP server, ignoring openHttp parameter"); + } + + // Use standard opening + return RocksDB.open(options, dataPath); + } + + @Override + protected RocksDB doOpenRocksDB(DBOptions dbOptions, String dataPath, + List cfDescriptors, + List cfHandles) throws RocksDBException { + try { + return RocksDB.open(dbOptions, dataPath, cfDescriptors, cfHandles); + } catch (RocksDBException e) { + throw e; + } + } + + @Override + protected RocksDB doOpenRocksDB(DBOptions dbOptions, String dataPath, + List cfDescriptors, + List cfHandles, + String optionPath, Boolean openHttp) throws RocksDBException { + // Standard RocksDB doesn't support optionPath and openHttp parameters + // Log a warning if these parameters are provided + if (optionPath != null && !optionPath.isEmpty()) { + LOG.warn("Standard RocksDB provider does not support optionPath parameter: {}", + optionPath); + } + if (openHttp != null && openHttp) { + LOG.warn("Standard RocksDB provider does not support openHttp parameter"); + } + + // Fallback to standard column family opening + return doOpenRocksDB(dbOptions, dataPath, cfDescriptors, cfHandles); + } + + @Override + public void initialize() { + try { + // Load RocksDB native library + RocksDB.loadLibrary(); + LOG.debug("Standard RocksDB library loaded successfully"); + } catch (Exception e) { + LOG.error("Failed to load standard RocksDB library", e); + throw new RuntimeException("Failed to initialize standard RocksDB provider", e); + } + } + + @Override + protected void performProviderSpecificClose(RocksDB rocksDB) { + // Standard RocksDB doesn't require special close operations + // The base class will handle the standard rocksDB.close() call + LOG.debug("Standard RocksDB close completed"); + } + + @Override + public void shutdown() { + LOG.info("Standard RocksDB provider shutdown completed"); + } +} diff --git a/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/ToplingRocksDBProvider.java b/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/ToplingRocksDBProvider.java new file mode 100644 index 0000000000..c80ac4c7e7 --- /dev/null +++ b/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/ToplingRocksDBProvider.java @@ -0,0 +1,490 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.rocksdb.provider; + +import org.rocksdb.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.commons.lang3.StringUtils; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.Map; + +import net.minidev.json.JSONObject; + +/** + * ToplingRocksDBProvider provides ToplingDB-specific RocksDB functionality. + * This provider supports advanced ToplingDB features including: + * - YAML-based configuration via optionPath + * - HTTP server for monitoring and management + * - SidePluginRepo integration for enhanced performance + */ +public class ToplingRocksDBProvider extends AbstractRocksDBProvider { + + private static final Logger LOG = LoggerFactory.getLogger(ToplingRocksDBProvider.class); + + private static final String PROVIDER_NAME = "topling"; + private static final int PROVIDER_PRIORITY = 200; // Higher priority than standard + private static final String SIDE_PLUGIN_REPO_CLASS = "org.rocksdb.SidePluginRepo"; + + // Store repo objects for proper cleanup + private final Map rocksDBToRepoMap = new ConcurrentHashMap<>(); + + @Override + public String getProviderName() { + return PROVIDER_NAME; + } + + @Override + public int getPriority() { + return PROVIDER_PRIORITY; + } + + @Override + public boolean isAvailable() { + try { + // Check if SidePluginRepo class is available + Class.forName(SIDE_PLUGIN_REPO_CLASS); + LOG.info("ToplingDB SidePluginRepo found, ToplingRocksDBProvider is available"); + return true; + } catch (ClassNotFoundException e) { + LOG.debug( + "ToplingDB SidePluginRepo not found, ToplingRocksDBProvider is not available:" + + " {}", + e.getMessage()); + return false; + } + } + + @Override + protected RocksDB doOpenRocksDB(Options options, String dataPath) throws RocksDBException { + LOG.info("Opening RocksDB with Options at path: {}", dataPath); + + // For simple Options-based opening without optionPath, use standard RocksDB.open + return RocksDB.open(options, dataPath); + } + + @Override + public RocksDB openRocksDB(Options options, String dataPath, String optionPath, + Boolean openHttp) throws RocksDBException { + return doOpenRocksDB(options, dataPath, optionPath, openHttp); + } + + @Override + protected RocksDB doOpenRocksDB(Options options, String dataPath, String optionPath, + Boolean openHttp) throws RocksDBException { + // Check if we should use ToplingDB features + boolean useTopling = validateConfiguration(optionPath); + + if (useTopling) { + return openWithToplingFeatures(options, dataPath, optionPath, openHttp); + } else { + LOG.warn( + "optionPath: {} is not ToplingDB configuration, opening RocksDB without " + + "ToplingDB features", + optionPath); + return RocksDB.open(options, dataPath); + } + } + + @Override + protected RocksDB doOpenRocksDB(DBOptions dbOptions, String dataPath, + List cfDescriptors, + List cfHandles) throws RocksDBException { + LOG.info("Opening RocksDB with DBOptions and column families at path: {}", dataPath); + + // For column family opening without ToplingDB features, use standard RocksDB.open + return RocksDB.open(dbOptions, dataPath, cfDescriptors, cfHandles); + } + + @Override + protected RocksDB doOpenRocksDB(DBOptions dbOptions, String dataPath, + List cfDescriptors, + List cfHandles, + String optionPath, Boolean openHttp) throws RocksDBException { + // Check if we should use ToplingDB features + boolean useTopling = validateConfiguration(optionPath); + + if (useTopling) { + // For ToplingDB with column families, we need to use the standard RocksDB.open method + // but with ToplingDB-specific initialization through SidePluginRepo + return openWithToplingFeaturesAndCF(dbOptions, dataPath, cfDescriptors, cfHandles, + optionPath, openHttp); + } else { + LOG.warn( + "optionPath: {} is not ToplingDB configuration, opening RocksDB without " + + "ToplingDB features", + optionPath); + return RocksDB.open(dbOptions, dataPath, cfDescriptors, cfHandles); + } + } + + /** + * Opens RocksDB using ToplingDB features with SidePluginRepo + */ + private RocksDB openWithToplingFeatures(Options options, String dataPath, + String optionPath, Boolean openHttp) + throws RocksDBException { + try { + // Initialize ToplingDB repo with common operations + Object repo = initializeToplingRepo(options, dataPath, optionPath); + + // Open database with default column families + Class sidePluginRepoClass = repo.getClass(); + Method openDBMethod = sidePluginRepoClass.getMethod("openDB", String.class); + Object result = openDBMethod.invoke(repo, converseOptionsToJsonString(dataPath, null)); + + // Start HTTP server if needed + startHttpServerIfNeeded(repo, dataPath, openHttp, optionPath); + + // Validate and store result + return validateAndStoreResult(result, repo, dataPath, 0); + + } catch (InvocationTargetException e) { + Throwable cause = e.getCause(); + if (cause instanceof RocksDBException) { + throw (RocksDBException) cause; + } else { + throw new RocksDBException( + "Failed to open DB with SidePluginRepo: " + cause.getMessage()); + } + } catch (Exception e) { + throw new RocksDBException("Failed to open ToplingDB: " + e.getMessage()); + } + } + + /** + * Open RocksDB with ToplingDB features and column families support + */ + private RocksDB openWithToplingFeaturesAndCF(DBOptions dbOptions, String dataPath, + List cfDescriptors, + List cfHandles, + String optionPath, Boolean openHttp) + throws RocksDBException { + + try { + // Initialize ToplingDB repo with common operations + Object repo = initializeToplingRepo(dbOptions, dataPath, optionPath); + + // Prepare column family names for JSON + List cfNames = new java.util.ArrayList<>(); + for (ColumnFamilyDescriptor cfDescriptor : cfDescriptors) { + cfNames.add(new String(cfDescriptor.getName())); + } + + // Open database with column families + Class sidePluginRepoClass = repo.getClass(); + Method openDBMethod = sidePluginRepoClass.getMethod("openDB", String.class, List.class); + Object result = openDBMethod.invoke(repo, + converseOptionsToJsonString(dataPath, cfNames), + cfHandles); + + // Start HTTP server if needed + startHttpServerIfNeeded(repo, dataPath, openHttp, optionPath); + + // Validate and store result + return validateAndStoreResult(result, repo, dataPath, cfDescriptors.size()); + + } catch (InvocationTargetException e) { + Throwable cause = e.getCause(); + if (cause instanceof RocksDBException) { + throw (RocksDBException) cause; + } else { + throw new RocksDBException( + "Failed to open DB with SidePluginRepo: " + cause.getMessage()); + } + } catch (Exception e) { + LOG.error("Failed to open ToplingDB with column families", e); + throw new RocksDBException("Failed to open ToplingDB: " + e.getMessage()); + } + } + + /** + * Common operations for ToplingDB SidePluginRepo initialization and setup + */ + private Object initializeToplingRepo(Object options, String dataPath, String optionPath) + throws RocksDBException { + try { + // Dynamically load the SidePluginRepo class by its name at runtime. + Class sidePluginRepoClass = Class.forName(SIDE_PLUGIN_REPO_CLASS); + Object repo = sidePluginRepoClass.getConstructor().newInstance(); + + String dbName = getDbName(dataPath); + + // Put options into repo - handle both Options and DBOptions + if (options instanceof Options) { + Method putMethod = + sidePluginRepoClass.getMethod("put", String.class, Options.class); + putMethod.invoke(repo, dbName, options); + } else if (options instanceof DBOptions) { + Method putMethod = + sidePluginRepoClass.getMethod("put", String.class, DBOptions.class); + putMethod.invoke(repo, dbName, options); + } else { + throw new RocksDBException( + "Unsupported options type: " + options.getClass().getName()); + } + + // Import auto file + Method importAutoFileMethod = + sidePluginRepoClass.getMethod("importAutoFile", String.class); + importAutoFileMethod.invoke(repo, optionPath); + + return repo; + + } catch (ClassNotFoundException e) { + LOG.error( + "SidePluginRepo not found. This version of rocksdbjni does not support " + + "topling.", + e); + throw new IllegalStateException( + "Topling features (SidePluginRepo) are required but not found in the " + + "rocksdbjni library.", + e); + } catch (Exception e) { + LOG.error("Failed to initialize ToplingDB SidePluginRepo", e); + throw new RocksDBException("SidePluginRepo reflection error: " + e.getMessage()); + } + } + + /** + * Start HTTP server if conditions are met + */ + private void startHttpServerIfNeeded(Object repo, String dataPath, Boolean openHttp, + String optionPath) + throws RocksDBException { + try { + if (Boolean.TRUE.equals(openHttp)) { + Class sidePluginRepoClass = repo.getClass(); + Method openHttpMethod = sidePluginRepoClass.getMethod("startHttpServer"); + openHttpMethod.invoke(repo); + LOG.info("Topling HTTP Server has been started according to the " + + "listening_ports specified in " + optionPath); + } + } catch (Exception e) { + LOG.error("Failed to start HTTP server", e); + throw new RocksDBException("Failed to start HTTP server: " + e.getMessage()); + } + } + + /** + * Validate and store RocksDB result with repo mapping + */ + private RocksDB validateAndStoreResult(Object result, Object repo, String dataPath, int cfCount) + throws RocksDBException { + if (result instanceof RocksDB) { + RocksDB rocksDB = (RocksDB) result; + // Store the repo reference for later cleanup + rocksDBToRepoMap.put(rocksDB, repo); + if (cfCount > 0) { + LOG.info("Successfully opened ToplingDB with {} column families at path: {}", + cfCount, dataPath); + } else { + LOG.info("Successfully opened ToplingDB with default column families at path: {}", + dataPath); + } + return rocksDB; + } else { + throw new RocksDBException("ToplingDB openDB returned unexpected result type: " + + (result != null ? result.getClass().getName() : "null")); + } + } + + /** + * Utility function to convert options to JSON string for ToplingDB + * Moved from RocksDBStdSessions + */ + private static String converseOptionsToJsonString(String dataPath, List cfs) { + if (dataPath == null || dataPath.trim().isEmpty()) { + throw new IllegalArgumentException("dataPath cannot be null or empty"); + } + // sanitize path to avoid trailing slash causing empty namepart in native side + String sanitizedPath = sanitizePath(dataPath); + // construct CFOptions + JSONObject columnFamilies = new JSONObject(); + // multi CFs + if (cfs != null) { + for (String cf : cfs) { + columnFamilies.put(cf, "$default"); + } + } else { // single default CF + columnFamilies.put("default", "$default"); + } + + // construct params + JSONObject params = new JSONObject(); + params.put("db_options", "$dbo"); + params.put("cf_options", "$default"); + params.put("column_families", columnFamilies); + params.put("path", sanitizedPath); + + // construct wrapper + JSONObject wrapper = new JSONObject(); + wrapper.put("method", "DB::Open"); + wrapper.put("params", params); + + return wrapper.toString(); + } + + /** + * Utility function to get database name from path + * Moved from RocksDBStdSessions + */ + private static String getDbName(String dataPath) { + String sanitizedPath = sanitizePath(dataPath); + return Paths.get(sanitizedPath).getFileName().toString(); + } + + /** + * Ensure path has no trailing separators and is normalized + */ + private static String sanitizePath(String dataPath) { + String p = dataPath.trim(); + // remove trailing separators + while (p.endsWith("/") || p.endsWith(java.io.File.separator)) { + p = p.substring(0, p.length() - 1); + } + // normalize using Paths to collapse redundant parts + try { + return Paths.get(p).normalize().toString(); + } catch (Exception e) { + // fallback to original trimmed path + return p; + } + } + + private static boolean validateConfiguration(String optionPath) { + boolean result = false; + if (!StringUtils.isEmpty(optionPath)) { + try { + // Validate option path first + validateOptionPath(optionPath); + + // Read and validate YAML content + String yamlContent = Files.readString(Paths.get(optionPath)); + validateYamlContent(yamlContent); + + Class.forName(SIDE_PLUGIN_REPO_CLASS); + result = true; + LOG.info( + "SidePluginRepo found. Will attempt to open default CF RocksDB using " + + "Topling."); + } catch (ClassNotFoundException e) { + LOG.warn("SidePluginRepo not found, even though 'optionPath' was provided. " + + "Falling back to the standard RocksDB default CF opening method. " + + "The configuration in '{}' will be ignored.", optionPath); + } catch (Exception e) { + LOG.warn( + "Failed to validate optionPath '{}': {}. Falling back to standard RocksDB.", + optionPath, e.getMessage()); + } + } + return result; + } + + /** + * Validate option path using RocksDBOptions utility method. + * This method will be available when hugegraph-rocksdb is in classpath. + */ + private static void validateOptionPath(String optionPath) { + try { + // Use reflection to call RocksDBOptions.validateOptionPath + Class rocksDBOptionsClass = + Class.forName("org.apache.hugegraph.backend.store.rocksdb.RocksDBOptions"); + Method validateMethod = + rocksDBOptionsClass.getMethod("validateOptionPath", String.class); + validateMethod.invoke(null, optionPath); + } catch (Exception e) { + // Fallback to basic validation if RocksDBOptions is not available + LOG.warn("RocksDBOptions.validateOptionPath not available, using basic validation: {}", + e.getMessage()); + if (optionPath == null || optionPath.isBlank()) { + throw new IllegalArgumentException("option_path can't be null or empty"); + } + if (!optionPath.toLowerCase().endsWith(".yaml") && + !optionPath.toLowerCase().endsWith(".yml")) { + throw new IllegalArgumentException("option_path must end with .yaml or .yml"); + } + } + } + + /** + * Validate YAML content using RocksDBOptions utility method. + * This method will be available when hugegraph-rocksdb is in classpath. + */ + private static void validateYamlContent(String yamlContent) { + try { + // Use reflection to call RocksDBOptions.validateYamlContent + Class rocksDBOptionsClass = + Class.forName("org.apache.hugegraph.backend.store.rocksdb.RocksDBOptions"); + Method validateMethod = + rocksDBOptionsClass.getMethod("validateYamlContent", String.class); + validateMethod.invoke(null, yamlContent); + } catch (Exception e) { + // Fallback to basic validation if RocksDBOptions is not available + LOG.warn( + "RocksDBOptions.validateYamlContent not available, skipping YAML validation: " + + "{}", + e.getMessage()); + } + } + + @Override + public void initialize() { + LOG.info("Initializing ToplingRocksDBProvider"); + // ToplingDB-specific initialization if needed + } + + @Override + protected void performProviderSpecificClose(RocksDB rocksDB) { + LOG.info("Performing ToplingDB-specific cleanup for RocksDB instance"); + + // Get the repo object associated with this RocksDB instance + Object repo = rocksDBToRepoMap.remove(rocksDB); + + if (repo != null) { + LOG.info("SidePluginRepo instance found, attempting to call closeAllDB()."); + try { + // Get the class of the repo object at runtime. + Class sidePluginRepoClass = repo.getClass(); + Method closeAllDBMethod = sidePluginRepoClass.getMethod("closeAllDB"); + + // Invoke the method on the repo instance. + closeAllDBMethod.invoke(repo); + LOG.info("Successfully called closeAllDB() on SidePluginRepo."); + } catch (Exception e) { + // Catch potential reflection exceptions (e.g., NoSuchMethodException) + // and log them. This is safer than letting them crash the application. + LOG.error("Failed to reflectively call closeAllDB() on SidePluginRepo.", e); + } + } else { + LOG.debug("No SidePluginRepo found for this RocksDB instance, using standard close."); + } + } + + @Override + public void shutdown() { + LOG.info("Shutting down ToplingRocksDBProvider"); + // ToplingDB-specific shutdown if needed + } +} diff --git a/hugegraph-rocksdb-provider/src/main/resources/META-INF/services/org.apache.hugegraph.rocksdb.provider.RocksDBProvider b/hugegraph-rocksdb-provider/src/main/resources/META-INF/services/org.apache.hugegraph.rocksdb.provider.RocksDBProvider new file mode 100644 index 0000000000..590d4233a9 --- /dev/null +++ b/hugegraph-rocksdb-provider/src/main/resources/META-INF/services/org.apache.hugegraph.rocksdb.provider.RocksDBProvider @@ -0,0 +1,2 @@ +org.apache.hugegraph.rocksdb.provider.StandardRocksDBProvider +org.apache.hugegraph.rocksdb.provider.ToplingRocksDBProvider diff --git a/pom.xml b/pom.xml index 45f5a34a43..aea17eab39 100644 --- a/pom.xml +++ b/pom.xml @@ -103,6 +103,7 @@ hugegraph-pd hugegraph-store hugegraph-commons + hugegraph-rocksdb-provider install-dist hugegraph-cluster-test hugegraph-struct From 061460cefea6fa06b7f7c44366ad21e0e0a1e0ba Mon Sep 17 00:00:00 2001 From: gydeng Date: Tue, 4 Nov 2025 08:56:58 +0000 Subject: [PATCH 37/51] feat(server): refactor server using rocksdb-provider to make it support toplingdb --- hugegraph-server/hugegraph-rocksdb/pom.xml | 6 +- .../backend/store/rocksdb/OpenedRocksDB.java | 24 +- .../store/rocksdb/RocksDBStdSessions.java | 282 ++---------------- .../backend/store/rocksdb/RocksDBStore.java | 26 +- .../store/rocksdbsst/RocksDBSstStore.java | 3 +- 5 files changed, 46 insertions(+), 295 deletions(-) diff --git a/hugegraph-server/hugegraph-rocksdb/pom.xml b/hugegraph-server/hugegraph-rocksdb/pom.xml index c749254962..8c4f6618e9 100644 --- a/hugegraph-server/hugegraph-rocksdb/pom.xml +++ b/hugegraph-server/hugegraph-rocksdb/pom.xml @@ -35,9 +35,9 @@ ${revision} - org.rocksdb - rocksdbjni - 8.10.2-SNAPSHOT + org.apache.hugegraph + hugegraph-rocksdb-provider + ${revision} diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/OpenedRocksDB.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/OpenedRocksDB.java index e83ba04dc4..daae523524 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/OpenedRocksDB.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/OpenedRocksDB.java @@ -19,7 +19,6 @@ import java.io.File; import java.io.IOException; -import java.lang.reflect.Method; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Map; @@ -32,6 +31,7 @@ import org.apache.hugegraph.backend.store.rocksdb.RocksDBIteratorPool.ReusedRocksIterator; import org.apache.hugegraph.util.E; import org.apache.hugegraph.util.Log; +import org.apache.hugegraph.rocksdb.provider.RocksDBProviderLoader; import org.rocksdb.Checkpoint; import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.RocksDB; @@ -45,14 +45,12 @@ public class OpenedRocksDB implements AutoCloseable { private final RocksDB rocksdb; private final Map cfHandles; private final SstFileManager sstFileManager; - private final Object repo; public OpenedRocksDB(RocksDB rocksdb, Map cfHandles, - SstFileManager sstFileManager, Object repo) { + SstFileManager sstFileManager) { this.rocksdb = rocksdb; this.cfHandles = cfHandles; this.sstFileManager = sstFileManager; - this.repo = repo; } protected final RocksDB rocksdb() { @@ -93,22 +91,8 @@ public void close() { } this.cfHandles.clear(); - if (repo != null) { - LOG.info("SidePluginRepo instance found, attempting to call closeAllDB()."); - try { - // Get the class of the repo object at runtime. - Class sidePluginRepoClass = repo.getClass(); - Method closeAllDBMethod = sidePluginRepoClass.getMethod("closeAllDB"); - - // Invoke the method on the repo instance. - closeAllDBMethod.invoke(repo); - } catch (Exception e) { - // Catch potential reflection exceptions (e.g., NoSuchMethodException) - // and log them. This is safer than letting them crash the application. - LOG.error("Failed to reflectively call closeAllDB() on SidePluginRepo.", e); - } - } - this.rocksdb.close(); + // Use RocksDBProviderLoader to close RocksDB + RocksDBProviderLoader.closeRocksDB(this.rocksdb); } public long totalSize() { diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java index 9853eab7e4..b1307af572 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java @@ -18,9 +18,6 @@ package org.apache.hugegraph.backend.store.rocksdb; import java.io.File; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -34,7 +31,6 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.hugegraph.backend.BackendException; import org.apache.hugegraph.backend.serializer.BinarySerializer; @@ -49,6 +45,7 @@ import org.apache.hugegraph.util.E; import org.apache.hugegraph.util.Log; import org.apache.hugegraph.util.StringEncoding; +import org.apache.hugegraph.rocksdb.provider.RocksDBProviderLoader; import org.rocksdb.BlockBasedTableConfig; import org.rocksdb.BloomFilter; import org.rocksdb.ColumnFamilyDescriptor; @@ -76,8 +73,6 @@ import com.google.common.collect.ImmutableList; -import net.minidev.json.JSONObject; - public class RocksDBStdSessions extends RocksDBSessions { private static final Logger LOG = Log.logger(RocksDBStdSessions.class); @@ -85,79 +80,33 @@ public class RocksDBStdSessions extends RocksDBSessions { private final HugeConfig config; private final String dataPath; private final String walPath; - private final String optionPath; - private final Boolean openHttp; private volatile OpenedRocksDB rocksdb; private final AtomicInteger refCount; public RocksDBStdSessions(HugeConfig config, String database, String store, - String dataPath, String walPath, Object... optionalArgs) throws - RocksDBException { + String dataPath, String walPath) throws + RocksDBException { super(config, database, store); this.config = config; this.dataPath = dataPath; this.walPath = walPath; - // Parse optional Args - if (optionalArgs.length >= 2) { - if (!(optionalArgs[0] instanceof String)) { - throw new IllegalArgumentException("Expected String for optionPath at position 0"); - } - if (!(optionalArgs[1] instanceof Boolean)) { - throw new IllegalArgumentException("Expected Boolean for openHttp at position 1"); - } - this.optionPath = (String) optionalArgs[0]; - this.openHttp = (Boolean) optionalArgs[1]; - } else if (optionalArgs.length == 1) { - if (!(optionalArgs[0] instanceof String)) { - throw new IllegalArgumentException("Expected String for optionPath at position 0"); - } - this.optionPath = (String) optionalArgs[0]; - this.openHttp = false; - } else { - this.optionPath = null; - this.openHttp = false; - } - - this.rocksdb = RocksDBStdSessions.openRocksDB(config, dataPath, walPath, optionPath, - openHttp); + this.rocksdb = RocksDBStdSessions.openRocksDB(config, dataPath, walPath); this.refCount = new AtomicInteger(1); } public RocksDBStdSessions(HugeConfig config, String database, String store, String dataPath, String walPath, - List cfNames, Object... optionalArgs) throws - RocksDBException { + List cfNames) throws + RocksDBException { super(config, database, store); this.config = config; this.dataPath = dataPath; this.walPath = walPath; - // Parse optional Args - if (optionalArgs.length >= 2) { - if (!(optionalArgs[0] instanceof String)) { - throw new IllegalArgumentException("Expected String for optionPath at position 0"); - } - if (!(optionalArgs[1] instanceof Boolean)) { - throw new IllegalArgumentException("Expected Boolean for openHttp at position 1"); - } - this.optionPath = (String) optionalArgs[0]; - this.openHttp = (Boolean) optionalArgs[1]; - } else if (optionalArgs.length == 1) { - if (!(optionalArgs[0] instanceof String)) { - throw new IllegalArgumentException("Expected String for optionPath at position 0"); - } - this.optionPath = (String) optionalArgs[0]; - this.openHttp = false; - } else { - this.optionPath = null; - this.openHttp = false; - } - this.rocksdb = - RocksDBStdSessions.openRocksDB(config, cfNames, dataPath, walPath, optionPath, - openHttp); + RocksDBStdSessions.openRocksDB(config, cfNames, dataPath, walPath); this.refCount = new AtomicInteger(1); this.ingestExternalFile(); @@ -169,8 +118,6 @@ private RocksDBStdSessions(HugeConfig config, String database, String store, this.config = config; this.dataPath = origin.dataPath; this.walPath = origin.walPath; - this.optionPath = origin.optionPath; - this.openHttp = origin.openHttp; this.rocksdb = origin.rocksdb; this.refCount = origin.refCount; this.refCount.incrementAndGet(); @@ -267,8 +214,7 @@ public void reloadRocksDB() throws RocksDBException { this.rocksdb.close(); } this.rocksdb = RocksDBStdSessions.openRocksDB(this.config, ImmutableList.of(), - this.dataPath, this.walPath, - this.optionPath, this.openHttp); + this.dataPath, this.walPath); } @Override @@ -362,8 +308,7 @@ public String buildSnapshotPath(String snapshotPrefix) { public String hardLinkSnapshot(String snapshotPath) throws RocksDBException { String snapshotLinkPath = this.dataPath + "_temp"; try (OpenedRocksDB rocksdb = openRocksDB(this.config, ImmutableList.of(), - snapshotPath, null, this.optionPath, - this.openHttp)) { + snapshotPath, null)) { rocksdb.createCheckpoint(snapshotLinkPath); } LOG.info("The snapshot {} has been hard linked to {}", snapshotPath, snapshotLinkPath); @@ -428,101 +373,31 @@ private void ingestExternalFile() throws RocksDBException { } private static OpenedRocksDB openRocksDB(HugeConfig config, String dataPath, - String walPath, String optionPath, - Boolean openHttp) throws - RocksDBException { + String walPath) throws + RocksDBException { // Init options Options options = new Options(); RocksDBStdSessions.initOptions(config, options, options, options, options); options.setWalDir(walPath); SstFileManager sstFileManager = new SstFileManager(Env.getDefault()); options.setSstFileManager(sstFileManager); - /* - * Open RocksDB at the first time - * Don't merge old CFs, we expect a clear DB when using this one - */ - RocksDB rocksdb = null; - Object repo = null; - - boolean useTopling = false; - if (!StringUtils.isEmpty(optionPath)) { - try { - RocksDBOptions.validateOptionPath(optionPath); - RocksDBOptions.validateYamlContent(optionPath); - Class.forName("org.rocksdb.SidePluginRepo"); - useTopling = true; - LOG.info("SidePluginRepo found. Will attempt to open default CF RocksDB using " + - "Topling."); - } catch (ClassNotFoundException e) { - LOG.warn("SidePluginRepo not found, even though 'optionPath' was provided. " + - "Falling back to the standard RocksDB default CF opening method. " + - "The configuration in '{}' will be ignored.", optionPath); - } - } - if (useTopling) { - try { - // Dynamically load the SidePluginRepo class by its name at runtime. - Class sidePluginRepoClass = Class.forName("org.rocksdb.SidePluginRepo"); - - repo = sidePluginRepoClass.getConstructor().newInstance(); - String dbName = getDbName(dataPath); - Method putMethod = - sidePluginRepoClass.getMethod("put", String.class, Options.class); - putMethod.invoke(repo, dbName, options); - Method importAutoFileMethod = - sidePluginRepoClass.getMethod("importAutoFile", String.class); - importAutoFileMethod.invoke(repo, optionPath); - - Method openDBMethod = sidePluginRepoClass.getMethod("openDB", String.class); - rocksdb = (RocksDB) openDBMethod.invoke(repo, converseOptionsToJsonString(dataPath, - null)); - - if (Boolean.TRUE.equals(openHttp) && - BackendStoreProvider.GRAPH_STORE.equals(dbName)) { - Method openHttpMethod = - sidePluginRepoClass.getMethod("startHttpServer"); - openHttpMethod.invoke(repo); - LOG.info("Topling HTTP Server has been started according to the " + - "listening_ports specified in " + optionPath); - } - } catch (ClassNotFoundException e) { - // CRITICAL: If the class is not found, the current rocksdbjni library does not - // include SidePluginRepo. - LOG.error( - "SidePluginRepo not found. This version of rocksdbjni does not support " + - "topling.", - e); - // Since the user provided an optionPath, the intent was to use a feature that is - // unavailable. Throwing an exception is the correct course of action. - throw new IllegalStateException( - "Topling features (SidePluginRepo) are required but not found in the " + - "rocksdbjni library.", - e); - } catch (InvocationTargetException e) { - Throwable cause = e.getCause(); - if (cause instanceof RocksDBException) { - throw (RocksDBException) cause; - } else { - throw new RocksDBException( - "Failed to open DB with SidePluginRepo: " + cause.getMessage()); - } - } catch (InstantiationException | NoSuchMethodException | IllegalAccessException e) { - throw new RocksDBException("SidePluginRepo reflection error: " + e.getMessage()); - } - } else { - rocksdb = RocksDB.open(options, dataPath); - } + // Use RocksDBProviderLoader to open RocksDB + // Only enable HTTP server for GRAPH_STORE when openHttp is true + boolean openHttp = Boolean.TRUE.equals(config.get(RocksDBOptions.OPEN_HTTP)) && + BackendStoreProvider.GRAPH_STORE.equals(getDbName(dataPath)); + RocksDB rocksdb = RocksDBProviderLoader.openRocksDB(options, dataPath, + config.get(RocksDBOptions.OPTION_PATH), + openHttp); Map cfs = new ConcurrentHashMap<>(); - return new OpenedRocksDB(rocksdb, cfs, sstFileManager, repo); + return new OpenedRocksDB(rocksdb, cfs, sstFileManager); } private static OpenedRocksDB openRocksDB(HugeConfig config, List cfNames, String dataPath, - String walPath, String optionPath, - Boolean openHttp) throws - RocksDBException { + String walPath) throws + RocksDBException { // Old CFs should always be opened Set mergedCFs = RocksDBStdSessions.mergeOldCFs(dataPath, cfNames); @@ -548,82 +423,13 @@ private static OpenedRocksDB openRocksDB(HugeConfig config, // Open RocksDB with CFs List cfhs = new ArrayList<>(); - RocksDB rocksdb = null; - Object repo = null; - - boolean useTopling = false; - if (!StringUtils.isEmpty(optionPath)) { - try { - RocksDBOptions.validateOptionPath(optionPath); - RocksDBOptions.validateYamlContent(optionPath); - Class.forName("org.rocksdb.SidePluginRepo"); - useTopling = true; - LOG.info("SidePluginRepo found. Will attempt to open multi CFs RocksDB using " + - "Topling plugin."); - } catch (ClassNotFoundException e) { - LOG.warn("SidePluginRepo not found, even though 'optionPath' was provided. " + - "Falling back to the standard RocksDB opening multi CFs method. " + - "The configuration in '{}' will be ignored.", optionPath); - } - } - - if (useTopling) { - try { - // Use reflection to check for the existence of the SidePluginRepo class at - // runtime. - Class sidePluginRepoClass = Class.forName("org.rocksdb.SidePluginRepo"); - - // Get the constructor and create a new instance. - Constructor constructor = sidePluginRepoClass.getConstructor(); - repo = constructor.newInstance(); - - String dbName = getDbName(dataPath); - - Method putMethod = - sidePluginRepoClass.getMethod("put", String.class, DBOptions.class); - putMethod.invoke(repo, dbName, options); - - Method importAutoFileMethod = - sidePluginRepoClass.getMethod("importAutoFile", String.class); - importAutoFileMethod.invoke(repo, optionPath); - - Method openDBMethod = - sidePluginRepoClass.getMethod("openDB", String.class, List.class); - rocksdb = (RocksDB) openDBMethod.invoke(repo, - converseOptionsToJsonString(dataPath, cfs), - cfhs); - - if (Boolean.TRUE.equals(openHttp) && - BackendStoreProvider.GRAPH_STORE.equals(dbName)) { - Method openHttpMethod = - sidePluginRepoClass.getMethod("startHttpServer"); - openHttpMethod.invoke(repo); - LOG.info("Topling HTTP Server has been started according to the " + - "listening_ports specified in " + optionPath); - } - - LOG.info("Successfully opened DB with SidePluginRepo."); - } catch (ClassNotFoundException e) { - // In this case, this exception should not occur - throw new IllegalStateException( - "Topling features (SidePluginRepo) are required but not found in the " + - "rocksdbjni library.", - e); - } catch (InvocationTargetException e) { - Throwable cause = e.getCause(); - if (cause instanceof RocksDBException) { - throw (RocksDBException) cause; - } else { - throw new RocksDBException( - "Failed to open DB with SidePluginRepo: " + cause.getMessage()); - } - } catch (InstantiationException | NoSuchMethodException | IllegalAccessException e) { - throw new RocksDBException("SidePluginRepo reflection error: " + e.getMessage()); - } - } else { - // use rocksdb - rocksdb = RocksDB.open(options, dataPath, cfds, cfhs); - } + // Use RocksDBProviderLoader to open RocksDB + // Only enable HTTP server for GRAPH_STORE when openHttp is true + boolean openHttp = Boolean.TRUE.equals(config.get(RocksDBOptions.OPEN_HTTP)) && + BackendStoreProvider.GRAPH_STORE.equals(getDbName(dataPath)); + RocksDB rocksdb = RocksDBProviderLoader.openRocksDB(options, dataPath, cfds, cfhs, + config.get(RocksDBOptions.OPTION_PATH), + openHttp); E.checkState(cfhs.size() == cfs.size(), "Expect same size of cf-handles and cf-names"); @@ -632,7 +438,7 @@ private static OpenedRocksDB openRocksDB(HugeConfig config, for (int i = 0; i < cfs.size(); i++) { cfHandles.put(cfs.get(i), new OpenedRocksDB.CFHandle(rocksdb, cfhs.get(i))); } - return new OpenedRocksDB(rocksdb, cfHandles, sstFileManager, repo); + return new OpenedRocksDB(rocksdb, cfHandles, sstFileManager); } private static Set mergeOldCFs(String path, @@ -656,36 +462,6 @@ public static Set listCFs(String path) throws RocksDBException { return cfs; } - private static String converseOptionsToJsonString(String dataPath, List cfs) { - if (dataPath == null || dataPath.trim().isEmpty()) { - throw new IllegalArgumentException("dataPath cannot be null or empty"); - } - // construct CFOptions - JSONObject columnFamilies = new JSONObject(); - // multi CFs - if (cfs != null) { - for (String cf : cfs) { - columnFamilies.put(cf, "$default"); - } - } else { // single default CF - columnFamilies.put("default", "$default"); - } - - // construct params - JSONObject params = new JSONObject(); - params.put("db_options", "$dbo"); - params.put("cf_options", "$default"); - params.put("column_families", columnFamilies); - params.put("path", dataPath); - - // construct wrapper - JSONObject wrapper = new JSONObject(); - wrapper.put("method", "DB::Open"); - wrapper.put("params", params); - - return wrapper.toString(); - } - private static String getDbName(String dataPath) { return Paths.get(dataPath).getFileName().toString(); } diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStore.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStore.java index 2685abdf65..2117e1c24f 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStore.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStore.java @@ -240,9 +240,7 @@ public synchronized void open(HugeConfig config) { openedDisks.add(disk); List tables = this.tableNames(e.getKey()); futures.add(openPool.submit(() -> { - this.open(config, disk, disk, tables, - config.get(RocksDBOptions.OPTION_PATH), - config.get(RocksDBOptions.OPEN_HTTP)); + this.open(config, disk, disk, tables); })); } } @@ -303,19 +301,16 @@ private void shutdownOpenPool(ExecutorService openPool) { protected RocksDBSessions open(HugeConfig config, List tableNames) { String dataPath = this.wrapPath(config.get(RocksDBOptions.DATA_PATH)); String walPath = this.wrapPath(config.get(RocksDBOptions.WAL_PATH)); - String optionPath = config.get(RocksDBOptions.OPTION_PATH); - Boolean openHttp = config.get(RocksDBOptions.OPEN_HTTP); - return this.open(config, dataPath, walPath, tableNames, optionPath, openHttp); + return this.open(config, dataPath, walPath, tableNames); } protected RocksDBSessions open(HugeConfig config, String dataPath, - String walPath, List tableNames, String optionPath, - Boolean openHttp) { + String walPath, List tableNames) { LOG.info("Opening RocksDB with data path: {}", dataPath); RocksDBSessions sessions = null; try { sessions = this.openSessionPool(config, dataPath, - walPath, tableNames, optionPath, openHttp); + walPath, tableNames); } catch (RocksDBException e) { RocksDBSessions origin = this.dbs.get(dataPath); if (origin != null) { @@ -348,8 +343,7 @@ protected RocksDBSessions open(HugeConfig config, String dataPath, none = null; } try { - sessions = this.openSessionPool(config, dataPath, walPath, none, optionPath, - openHttp); + sessions = this.openSessionPool(config, dataPath, walPath, none); } catch (RocksDBException e1) { e = e1; } @@ -378,15 +372,13 @@ protected RocksDBSessions open(HugeConfig config, String dataPath, protected RocksDBSessions openSessionPool(HugeConfig config, String dataPath, String walPath, - List tableNames, String optionPath, - Boolean openHttp) throws - RocksDBException { + List tableNames) throws + RocksDBException { if (tableNames == null) { - return new RocksDBStdSessions(config, this.database, this.store, dataPath, walPath, - optionPath, openHttp); + return new RocksDBStdSessions(config, this.database, this.store, dataPath, walPath); } else { return new RocksDBStdSessions(config, this.database, this.store, - dataPath, walPath, tableNames, optionPath, openHttp); + dataPath, walPath, tableNames); } } diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdbsst/RocksDBSstStore.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdbsst/RocksDBSstStore.java index 9cab1b8ccb..d7b75610f1 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdbsst/RocksDBSstStore.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdbsst/RocksDBSstStore.java @@ -38,8 +38,7 @@ public RocksDBSstStore(final BackendStoreProvider provider, @Override protected RocksDBSessions openSessionPool(HugeConfig config, String dataPath, String walPath, - List tableNames, String optionPath, - Boolean openHttp) + List tableNames) throws RocksDBException { if (tableNames == null) { return new RocksDBSstSessions(config, this.database(), this.store(), dataPath); From 062ffb599d0bc0417564d88fd23391b64a3a8247 Mon Sep 17 00:00:00 2001 From: gydeng Date: Tue, 4 Nov 2025 10:24:34 +0000 Subject: [PATCH 38/51] feat(pd): refactor pd using rocksdb-provider to make it support toplingdb --- hugegraph-pd/hg-pd-cli/pom.xml | 2 +- hugegraph-pd/hg-pd-core/pom.xml | 8 +- .../apache/hugegraph/pd/config/PDConfig.java | 5 + .../hugegraph/pd/store/HgKVStoreImpl.java | 10 +- .../assembly/static/bin/start-hugegraph-pd.sh | 10 ++ .../src/assembly/static/bin/util.sh | 18 ++ .../src/assembly/static/conf/application.yml | 3 + .../static/conf/graphs/rocksdb_pd.yaml | 154 ++++++++++++++++++ 8 files changed, 203 insertions(+), 7 deletions(-) create mode 100644 hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_pd.yaml diff --git a/hugegraph-pd/hg-pd-cli/pom.xml b/hugegraph-pd/hg-pd-cli/pom.xml index 4920174d76..98b5954278 100644 --- a/hugegraph-pd/hg-pd-cli/pom.xml +++ b/hugegraph-pd/hg-pd-cli/pom.xml @@ -49,7 +49,7 @@ com.alipay.sofa jraft-core - 1.3.13 + 1.3.14 org.rocksdb diff --git a/hugegraph-pd/hg-pd-core/pom.xml b/hugegraph-pd/hg-pd-core/pom.xml index e17570d592..03988d8a0d 100644 --- a/hugegraph-pd/hg-pd-core/pom.xml +++ b/hugegraph-pd/hg-pd-core/pom.xml @@ -38,7 +38,7 @@ com.alipay.sofa jraft-core - 1.3.13 + 1.3.14 org.rocksdb @@ -47,9 +47,9 @@ - org.rocksdb - rocksdbjni - 6.29.5 + org.apache.hugegraph + hugegraph-rocksdb-provider + ${revision} org.apache.hugegraph diff --git a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/config/PDConfig.java b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/config/PDConfig.java index 5d6c8db5e5..3e68cf102c 100644 --- a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/config/PDConfig.java +++ b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/config/PDConfig.java @@ -58,6 +58,11 @@ public class PDConfig { @Value("${grpc.host}") private String host; + @Value("${rocksdb.option-path: ''}") + private String optionPath; + @Value("${rocksdb.open-http:false}") + private Boolean openHttp; + @Value("${license.verify-path}") private String verifyPath; @Value("${license.license-path}") diff --git a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/store/HgKVStoreImpl.java b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/store/HgKVStoreImpl.java index bd2e7a9e22..5d7ac2817f 100644 --- a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/store/HgKVStoreImpl.java +++ b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/store/HgKVStoreImpl.java @@ -36,6 +36,7 @@ import org.apache.hugegraph.pd.config.PDConfig; import org.apache.hugegraph.pd.grpc.Pdpb; import org.apache.hugegraph.pd.grpc.discovery.RegisterInfo; +import org.apache.hugegraph.rocksdb.provider.RocksDBProviderLoader; import org.rocksdb.Checkpoint; import org.rocksdb.Options; import org.rocksdb.ReadOptions; @@ -59,6 +60,8 @@ public class HgKVStoreImpl implements HgKVStore { private RocksDB db; private String dbPath; private Options dbOptions; + private String optionPath; + private Boolean openHttp; @Override public void init(PDConfig config) { @@ -76,6 +79,8 @@ public void init(PDConfig config) { log.warn("Failed to create data file,{}", e); } } + this.optionPath = config.getOptionPath(); + this.openHttp = config.getOpenHttp(); openRocksDB(dbPath); } catch (PDException e) { log.error("Failed to open data file,{}", e); @@ -327,14 +332,15 @@ public void close() { private void closeRocksDB() { if (this.db != null) { - this.db.close(); + RocksDBProviderLoader.closeRocksDB(this.db); this.db = null; } } private void openRocksDB(String dbPath) throws PDException { try { - this.db = RocksDB.open(dbOptions, dbPath); + this.db = RocksDBProviderLoader.openRocksDB(dbOptions, dbPath, this.optionPath, + this.openHttp); } catch (RocksDBException e) { log.error("Failed to open RocksDB from {}", dbPath, e); throw new PDException(Pdpb.ErrorType.ROCKSDB_LOAD_SNAPSHOT_ERROR_VALUE, e); diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh index b5d5346f34..c1696f9e37 100644 --- a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh +++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh @@ -59,9 +59,19 @@ PID_FILE="$BIN/pid" . "$BIN"/util.sh +PARENT_DIR="$(cd "$TOP"/../ && pwd)" +SERVER_VERSION_DIR="${SERVER_VERSION_DIR:-$(find_hugegraph_server_dir "$PARENT_DIR")}" +if [ -z "$SERVER_VERSION_DIR" ] || [ ! -d "$SERVER_VERSION_DIR" ]; then + echo "Error: failed to locate ${PARENT_DIR}/apache-hugegraph-server* . Set SERVER_VERSION_DIR manually." >&2 + exit 1 +fi + ensure_path_writable "$LOGS" ensure_path_writable "$PLUGINS" +# preload rocksdb/toplingdb +source "$SERVER_VERSION_DIR/bin/preload-topling.sh" + # The maximum and minimum heap memory that service can use MAX_MEM=$((32 * 1024)) MIN_MEM=$((1 * 512)) diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh index 5cc0c355be..8f6b8806fd 100644 --- a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh @@ -371,3 +371,21 @@ function kill_process_and_wait() { kill_process "$process_name" "$pid" wait_for_shutdown "$process_name" "$pid" "$timeout_s" } + +# Find HugeGraph server directory in parent path using prefix glob. +# Usage: find_hugegraph_server_dir "/path/to/parent" +# Returns: first matching directory path or empty string +function find_hugegraph_server_dir() { + local parent_dir="$1" + if [ -z "$parent_dir" ]; then + parent_dir="$(cd "${TOP:-$(pwd)}"/.. && pwd)" + fi + local found="" + for d in "$parent_dir"/apache-hugegraph-server*; do + if [ -d "$d" ]; then + found="$d" + break + fi + done + echo "$found" +} diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/application.yml b/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/application.yml index eb8fda4526..8828674874 100644 --- a/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/application.yml +++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/application.yml @@ -39,6 +39,9 @@ grpc: port: 8686 # The service address of grpc needs to be changed to the actual local IPv4 address when deploying. host: 127.0.0.1 +# rocksdb: + # option-path: ./conf/graphs/rocksdb_pd.yaml + # open-http: true server: # REST service port number diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_pd.yaml b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_pd.yaml new file mode 100644 index 0000000000..440c515056 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_pd.yaml @@ -0,0 +1,154 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# common parameters +http: + # normally parent path of db path + document_root: ./library + listening_ports: '127.0.0.1:2011' +setenv: + StrSimpleEnvNameNotOverwrite: StringValue + IntSimpleEnvNameNotOverwrite: 16384 + OverwriteThisEnv: + #comment: overwrite is default to false + overwrite: true + value: force overwrite this env by overwrite true +Cache: + lru_cache: + class: LRUCache + params: + capacity: 8G + num_shard_bits: -1 + strict_capacity_limit: false + high_pri_pool_ratio: 0.5 + use_adaptive_mutex: false + metadata_charge_policy: kFullChargeCacheMetadata +Statistics: + stat: + class: default + params: + discard_tickers: + - rocksdb.block.cache + - rocksdb.block.cachecompressed + - rocksdb.block + - rocksdb.memtable.payload.bytes.at.flush + - rocksdb.memtable.garbage.bytes.at.flush + - rocksdb.txn + - rocksdb.blobdb + - rocksdb.row.cache + - rocksdb.number.block + - rocksdb.bloom.filter + - rocksdb.persistent + - rocksdb.sim.block.cache + discard_histograms: + # comment: .... + - rocksdb.blobdb + - rocksdb.bytes.compressed + - rocksdb.bytes.decompressed + - rocksdb.num.index.and.filter.blocks.read.per.level + - rocksdb.num.data.blocks.read.per.level + - rocksdb.compression.times.nanos + - rocksdb.decompression.times.nanos + - rocksdb.read.block.get.micros + - rocksdb.write.raw.block.micros + # comment end of array + #stats_level: kAll + stats_level: kDisableAll +MemTableRepFactory: + cspp: + class: cspp + params: + mem_cap: 16G + use_vm: false + token_use_idle: true + chunk_size: 16K + convert_to_sst: kFileMmap + sync_sst_file: false + skiplist: + class: SkipList + params: + lookahead: 0 +TableFactory: + cspp_memtab_sst: + class: CSPPMemTabTable + params: # empty params + bb: + class: BlockBasedTable + params: + checksum: kCRC32c + block_size: 4K + block_restart_interval: 16 + index_block_restart_interval: 1 + metadata_block_size: 4K + enable_index_compression: true + block_cache: "${lru_cache}" + readers: + BlockBasedTable: bb + CSPPMemTabTable: cspp_memtab_sst + block_cache_compressed: + persistent_cache: + filter_policy: + dispatch: + class: DispatcherTable + params: + default: bb + readers: + BlockBasedTable: bb + CSPPMemTabTable: cspp_memtab_sst + level_writers: [ bb, bb, bb, bb, bb, bb ] +CFOptions: + default: + max_write_buffer_number: 6 + memtable_factory: "${cspp}" + write_buffer_size: 128M + # set target_file_size_base as small as 512K is to make many SST files, + # thus key prefix cache can present efficiency + target_file_size_base: 64M + target_file_size_multiplier: 1 + table_factory: dispatch + max_bytes_for_level_base: 512M + max_bytes_for_level_multiplier: 10 + level_compaction_dynamic_level_bytes: false + level0_slowdown_writes_trigger: 20 + level0_stop_writes_trigger: 36 + level0_file_num_compaction_trigger: 2 + merge_operator: uint64add # support merge + level_compaction_dynamic_file_size: true + optimize_filters_for_hits: true + allow_merge_memtables: true + min_write_buffer_number_to_merge: 2 + compression_per_level: + - kNoCompression + - kNoCompression + - kSnappyCompression + - kSnappyCompression + - kSnappyCompression + - kSnappyCompression + - kSnappyCompression +DBOptions: + dbo: + create_if_missing: true + create_missing_column_families: false # this is important, must be false to hugegraph + max_background_compactions: -1 + max_subcompactions: 4 + max_level1_subcompactions: 0 + inplace_update_support: false + WAL_size_limit_MB: 0 + statistics: "${stat}" + max_manifest_file_size: 100M + max_background_jobs: 8 + compaction_readahead_size: 0 + memtable_as_log_index: true From 0ac8aa6a8656586884634f2c52e7097d943b5232 Mon Sep 17 00:00:00 2001 From: gydeng Date: Tue, 4 Nov 2025 10:27:36 +0000 Subject: [PATCH 39/51] feat(store): refactor store using rocksdb-provider to make it support toplingdb --- .../static/conf/graphs/rocksdb_store.yaml | 154 ++++++++++++++++++ hugegraph-store/hg-store-core/pom.xml | 2 +- .../static/bin/start-hugegraph-store.sh | 8 + .../src/assembly/static/bin/util.sh | 18 ++ .../assembly/static/conf/application-pd.yml | 4 + .../node/metrics/RocksDBMetricsConst.java | 32 ++-- hugegraph-store/hg-store-rocksdb/pom.xml | 6 +- .../rocksdb/access/RocksDBOptions.java | 15 +- .../rocksdb/access/RocksDBSession.java | 10 +- 9 files changed, 224 insertions(+), 25 deletions(-) create mode 100644 hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_store.yaml diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_store.yaml b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_store.yaml new file mode 100644 index 0000000000..440c515056 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_store.yaml @@ -0,0 +1,154 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# common parameters +http: + # normally parent path of db path + document_root: ./library + listening_ports: '127.0.0.1:2011' +setenv: + StrSimpleEnvNameNotOverwrite: StringValue + IntSimpleEnvNameNotOverwrite: 16384 + OverwriteThisEnv: + #comment: overwrite is default to false + overwrite: true + value: force overwrite this env by overwrite true +Cache: + lru_cache: + class: LRUCache + params: + capacity: 8G + num_shard_bits: -1 + strict_capacity_limit: false + high_pri_pool_ratio: 0.5 + use_adaptive_mutex: false + metadata_charge_policy: kFullChargeCacheMetadata +Statistics: + stat: + class: default + params: + discard_tickers: + - rocksdb.block.cache + - rocksdb.block.cachecompressed + - rocksdb.block + - rocksdb.memtable.payload.bytes.at.flush + - rocksdb.memtable.garbage.bytes.at.flush + - rocksdb.txn + - rocksdb.blobdb + - rocksdb.row.cache + - rocksdb.number.block + - rocksdb.bloom.filter + - rocksdb.persistent + - rocksdb.sim.block.cache + discard_histograms: + # comment: .... + - rocksdb.blobdb + - rocksdb.bytes.compressed + - rocksdb.bytes.decompressed + - rocksdb.num.index.and.filter.blocks.read.per.level + - rocksdb.num.data.blocks.read.per.level + - rocksdb.compression.times.nanos + - rocksdb.decompression.times.nanos + - rocksdb.read.block.get.micros + - rocksdb.write.raw.block.micros + # comment end of array + #stats_level: kAll + stats_level: kDisableAll +MemTableRepFactory: + cspp: + class: cspp + params: + mem_cap: 16G + use_vm: false + token_use_idle: true + chunk_size: 16K + convert_to_sst: kFileMmap + sync_sst_file: false + skiplist: + class: SkipList + params: + lookahead: 0 +TableFactory: + cspp_memtab_sst: + class: CSPPMemTabTable + params: # empty params + bb: + class: BlockBasedTable + params: + checksum: kCRC32c + block_size: 4K + block_restart_interval: 16 + index_block_restart_interval: 1 + metadata_block_size: 4K + enable_index_compression: true + block_cache: "${lru_cache}" + readers: + BlockBasedTable: bb + CSPPMemTabTable: cspp_memtab_sst + block_cache_compressed: + persistent_cache: + filter_policy: + dispatch: + class: DispatcherTable + params: + default: bb + readers: + BlockBasedTable: bb + CSPPMemTabTable: cspp_memtab_sst + level_writers: [ bb, bb, bb, bb, bb, bb ] +CFOptions: + default: + max_write_buffer_number: 6 + memtable_factory: "${cspp}" + write_buffer_size: 128M + # set target_file_size_base as small as 512K is to make many SST files, + # thus key prefix cache can present efficiency + target_file_size_base: 64M + target_file_size_multiplier: 1 + table_factory: dispatch + max_bytes_for_level_base: 512M + max_bytes_for_level_multiplier: 10 + level_compaction_dynamic_level_bytes: false + level0_slowdown_writes_trigger: 20 + level0_stop_writes_trigger: 36 + level0_file_num_compaction_trigger: 2 + merge_operator: uint64add # support merge + level_compaction_dynamic_file_size: true + optimize_filters_for_hits: true + allow_merge_memtables: true + min_write_buffer_number_to_merge: 2 + compression_per_level: + - kNoCompression + - kNoCompression + - kSnappyCompression + - kSnappyCompression + - kSnappyCompression + - kSnappyCompression + - kSnappyCompression +DBOptions: + dbo: + create_if_missing: true + create_missing_column_families: false # this is important, must be false to hugegraph + max_background_compactions: -1 + max_subcompactions: 4 + max_level1_subcompactions: 0 + inplace_update_support: false + WAL_size_limit_MB: 0 + statistics: "${stat}" + max_manifest_file_size: 100M + max_background_jobs: 8 + compaction_readahead_size: 0 + memtable_as_log_index: true diff --git a/hugegraph-store/hg-store-core/pom.xml b/hugegraph-store/hg-store-core/pom.xml index 18abe4937f..793bc43502 100644 --- a/hugegraph-store/hg-store-core/pom.xml +++ b/hugegraph-store/hg-store-core/pom.xml @@ -55,7 +55,7 @@ com.alipay.sofa jraft-core - 1.3.13 + 1.3.14 org.rocksdb diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh index d8b965a07d..bbcbad6242 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh @@ -38,6 +38,12 @@ GITHUB="https://github.com" PID_FILE="$BIN/pid" . "$BIN"/util.sh +PARENT_DIR="$(cd "$TOP"/../ && pwd)" +SERVER_VERSION_DIR="${SERVER_VERSION_DIR:-$(find_hugegraph_server_dir "$PARENT_DIR")}" +if [ -z "$SERVER_VERSION_DIR" ] || [ ! -d "$SERVER_VERSION_DIR" ]; then + echo "Error: failed to locate ${PARENT_DIR}/apache-hugegraph-server* . Set SERVER_VERSION_DIR manually." >&2 + exit 1 +fi arch=$(uname -m) echo "Current arch: $arch" @@ -64,6 +70,8 @@ else echo "Unsupported architecture: $arch" fi +source "$SERVER_VERSION_DIR/bin/preload-topling.sh" + ##pd/store max user processes, ulimit -u # Reduce the maximum number of processes that can be opened by a normal dev/user export PROC_LIMITN=1024 diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh index 93b55311cb..a8e7b4f779 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh @@ -400,3 +400,21 @@ function kill_process_and_wait() { kill_process "$process_name" "$pid" wait_for_shutdown "$process_name" "$pid" "$timeout_s" } + +# Find HugeGraph server directory in parent path using prefix glob. +# Usage: find_hugegraph_server_dir "/path/to/parent" +# Returns: first matching directory path or empty string +function find_hugegraph_server_dir() { + local parent_dir="$1" + if [ -z "$parent_dir" ]; then + parent_dir="$(cd "${TOP:-$(pwd)}"/.. && pwd)" + fi + local found="" + for d in "$parent_dir"/apache-hugegraph-server*; do + if [ -d "$d" ]; then + found="$d" + break + fi + done + echo "$found" +} diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/conf/application-pd.yml b/hugegraph-store/hg-store-dist/src/assembly/static/conf/application-pd.yml index 0315c4b4fe..3788e0a89f 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/conf/application-pd.yml +++ b/hugegraph-store/hg-store-dist/src/assembly/static/conf/application-pd.yml @@ -32,3 +32,7 @@ rocksdb: write_buffer_size: 32000000 # For each rocksdb, the number of memtables reaches this value for writing to disk. min_write_buffer_number_to_merge: 16 + # Configuration file path for rocksdb/toplingdb + # option_path: ./conf/graphs/rocksdb_store.yaml + # Whether to start Topling's HTTP service + # open_http: false diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/RocksDBMetricsConst.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/RocksDBMetricsConst.java index 94bdc4c6bc..ceecc43399 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/RocksDBMetricsConst.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/RocksDBMetricsConst.java @@ -54,12 +54,12 @@ public final class RocksDBMetricsConst { TickerType.BLOCK_CACHE_INDEX_HIT, // Index cache hits. TickerType.BLOCK_CACHE_INDEX_ADD, // Index blocks added to cache. TickerType.BLOCK_CACHE_INDEX_BYTES_INSERT, // Bytes inserted into index cache. - TickerType.BLOCK_CACHE_INDEX_BYTES_EVICT, // Bytes evicted from index cache. + // BLOCK_CACHE_INDEX_BYTES_EVICT removed in RocksDB 8.x TickerType.BLOCK_CACHE_FILTER_MISS, // Filter cache misses. TickerType.BLOCK_CACHE_FILTER_HIT, // Filter cache hits. TickerType.BLOCK_CACHE_FILTER_ADD, // Filter blocks added to cache. TickerType.BLOCK_CACHE_FILTER_BYTES_INSERT, // Bytes inserted in filter cache. - TickerType.BLOCK_CACHE_FILTER_BYTES_EVICT, // Bytes evicted from filter cache. + // BLOCK_CACHE_FILTER_BYTES_EVICT removed in RocksDB 8.x TickerType.BLOCK_CACHE_DATA_MISS, // Data cache misses. TickerType.BLOCK_CACHE_DATA_HIT, // Data cache hits. TickerType.BLOCK_CACHE_DATA_ADD, // Data blocks added to cache. @@ -95,31 +95,31 @@ public final class RocksDBMetricsConst { TickerType.NUMBER_DB_NEXT_FOUND, // Number of successful next operations. TickerType.NUMBER_DB_PREV_FOUND, // Number of successful previous operations. TickerType.ITER_BYTES_READ, // Bytes read by iterators. - TickerType.NO_FILE_CLOSES, // Number of file close operations. + // NO_FILE_CLOSES removed in RocksDB 8.x TickerType.NO_FILE_OPENS, // Number of file open operations. TickerType.NO_FILE_ERRORS, // Number of file errors. TickerType.STALL_MICROS, // Time spent in a stall micro. TickerType.DB_MUTEX_WAIT_MICROS, // Time spent waiting on a mutex. - TickerType.RATE_LIMIT_DELAY_MILLIS, // Rate limiting delay in milliseconds. - TickerType.NO_ITERATORS, // Number of iterators created. + // RATE_LIMIT_DELAY_MILLIS removed in RocksDB 8.x + // NO_ITERATORS removed in RocksDB 8.x TickerType.NUMBER_MULTIGET_BYTES_READ, // Bytes read by multi-get operations. TickerType.NUMBER_MULTIGET_KEYS_READ, // Keys read in multi-get operations. TickerType.NUMBER_MULTIGET_CALLS, // Number of multi-get operations. - TickerType.NUMBER_FILTERED_DELETES, // Number of deletes filtered. + // NUMBER_FILTERED_DELETES removed in RocksDB 8.x TickerType.NUMBER_MERGE_FAILURES, // Number of merge failures. TickerType.BLOOM_FILTER_PREFIX_CHECKED, // Number of prefix bloom filter checks. TickerType.BLOOM_FILTER_PREFIX_USEFUL, // Number of useful prefix bloom filter checks. TickerType.NUMBER_OF_RESEEKS_IN_ITERATION, // Number of reseeks in iteration. TickerType.GET_UPDATES_SINCE_CALLS, // Number of get updates since calls. - TickerType.BLOCK_CACHE_COMPRESSED_MISS, // Misses in compressed block cache. - TickerType.BLOCK_CACHE_COMPRESSED_HIT, // Hits in compressed block cache. - TickerType.BLOCK_CACHE_COMPRESSED_ADD, // Compressed blocks added to cache. - TickerType.BLOCK_CACHE_COMPRESSED_ADD_FAILURES, // Failures adding compressed blocks. + // BLOCK_CACHE_COMPRESSED_MISS removed in RocksDB 8.x + // BLOCK_CACHE_COMPRESSED_HIT removed in RocksDB 8.x + // BLOCK_CACHE_COMPRESSED_ADD removed in RocksDB 8.x + // BLOCK_CACHE_COMPRESSED_ADD_FAILURES removed in RocksDB 8.x TickerType.WAL_FILE_SYNCED, // Number of synced WAL files. TickerType.WAL_FILE_BYTES, // Bytes written to WAL files. TickerType.WRITE_DONE_BY_SELF, // Writes completed by self. TickerType.WRITE_DONE_BY_OTHER, // Writes completed by others. - TickerType.WRITE_TIMEDOUT, // Number of write timeouts. + // WRITE_TIMEDOUT removed in RocksDB 8.x TickerType.WRITE_WITH_WAL, // Writes involving WAL. TickerType.COMPACT_READ_BYTES, // Bytes read during compaction. TickerType.COMPACT_WRITE_BYTES, // Bytes written during compaction. @@ -167,12 +167,10 @@ public final class RocksDBMetricsConst { // Time spent reading blocks during compaction. HistogramType.READ_BLOCK_GET_MICROS, // Time spent reading blocks during get. HistogramType.WRITE_RAW_BLOCK_MICROS, // Time spent writing raw blocks. - HistogramType.STALL_L0_SLOWDOWN_COUNT, // Count of stalls due to L0 slowdown. - HistogramType.STALL_MEMTABLE_COMPACTION_COUNT, - // Count of stalls due to memtable compaction. - HistogramType.STALL_L0_NUM_FILES_COUNT, // Count of stalls due to number of files at L0. - HistogramType.HARD_RATE_LIMIT_DELAY_COUNT, // Count of delays due to hard rate limits. - HistogramType.SOFT_RATE_LIMIT_DELAY_COUNT, // Count of delays due to soft rate limits. + // STALL_L0_SLOWDOWN_COUNT removed in RocksDB 8.x + // Note: The following constants were removed in RocksDB 8.10.2: + // STALL_MEMTABLE_COMPACTION_COUNT, STALL_L0_NUM_FILES_COUNT, + // HARD_RATE_LIMIT_DELAY_COUNT, SOFT_RATE_LIMIT_DELAY_COUNT HistogramType.NUM_FILES_IN_SINGLE_COMPACTION, // Number of files in a single compaction. HistogramType.DB_SEEK, // Latency of database seek operations. HistogramType.WRITE_STALL, // Time spent in write stalls. diff --git a/hugegraph-store/hg-store-rocksdb/pom.xml b/hugegraph-store/hg-store-rocksdb/pom.xml index bb463d7ed9..a6b3da361d 100644 --- a/hugegraph-store/hg-store-rocksdb/pom.xml +++ b/hugegraph-store/hg-store-rocksdb/pom.xml @@ -55,9 +55,9 @@ hg-store-common - org.rocksdb - rocksdbjni - 7.7.3 + org.apache.hugegraph + hugegraph-rocksdb-provider + ${revision} org.projectlombok diff --git a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBOptions.java b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBOptions.java index 7fcd07f3b8..77b635557d 100644 --- a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBOptions.java +++ b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBOptions.java @@ -56,7 +56,20 @@ public class RocksDBOptions extends OptionHolder { disallowEmpty(), false ); - + public static final ConfigOption OPTION_PATH = + new ConfigOption<>( + "rocksdb.option_path", + "The YAML file for configuring Topling/RocksDB parameters", + null, + "" + ); + public static final ConfigOption OPEN_HTTP = + new ConfigOption<>( + "rocksdb.open_http", + "Whether to start Topling's HTTP service", + disallowEmpty(), + false + ); // public static final ConfigListOption DATA_DISKS = // new ConfigListOption<>( // "rocksdb.data_disks", diff --git a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java index f4e7605a7f..ca40bf1271 100644 --- a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java +++ b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java @@ -38,6 +38,7 @@ import org.apache.commons.io.FileUtils; import org.apache.hugegraph.config.HugeConfig; import org.apache.hugegraph.rocksdb.access.util.Asserts; +import org.apache.hugegraph.rocksdb.provider.RocksDBProviderLoader; import org.apache.hugegraph.store.term.HgPair; import org.apache.hugegraph.util.Bytes; import org.apache.hugegraph.util.E; @@ -450,8 +451,11 @@ private void openRocksDB(String dbDataPath, long version) { new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOptions)); } List columnFamilyHandleList = new ArrayList<>(); - this.rocksDB = RocksDB.open(dbOptions, dbPath, columnFamilyDescriptorList, - columnFamilyHandleList); + this.rocksDB = + RocksDBProviderLoader.openRocksDB(dbOptions, dbPath, columnFamilyDescriptorList, + columnFamilyHandleList, + hugeConfig.get(RocksDBOptions.OPTION_PATH), + hugeConfig.get(RocksDBOptions.OPEN_HTTP)); Asserts.isTrue(columnFamilyHandleList.size() > 0, "must have column family"); for (ColumnFamilyHandle handle : columnFamilyHandleList) { @@ -636,7 +640,7 @@ void shutdown() { } catch (RocksDBException e) { log.warn("exception ", e); } - this.rocksDB.close(); + RocksDBProviderLoader.closeRocksDB(this.rocksDB); } rocksDB = null; if (dbOptions != null) { From 1ac8280e0b64661491b40d170ff0c7fece4b5125 Mon Sep 17 00:00:00 2001 From: gydeng Date: Tue, 4 Nov 2025 10:41:42 +0000 Subject: [PATCH 40/51] fix(pd): fix potential CI issue --- .github/workflows/pd-store-ci.yml | 3 +++ .../src/assembly/travis/install-rocksdb.sh | 26 ++++++++++++------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pd-store-ci.yml b/.github/workflows/pd-store-ci.yml index 4115e80892..30cd4558f5 100644 --- a/.github/workflows/pd-store-ci.yml +++ b/.github/workflows/pd-store-ci.yml @@ -106,6 +106,7 @@ jobs: - name: Prepare env and service run: | + source $TRAVIS_DIR/install-rocksdb.sh $TRAVIS_DIR/start-pd.sh - name: Run client test @@ -164,6 +165,7 @@ jobs: - name: Prepare env and service run: | + source $TRAVIS_DIR/install-rocksdb.sh $TRAVIS_DIR/start-pd.sh $TRAVIS_DIR/start-store.sh @@ -241,6 +243,7 @@ jobs: # todo remove --fail-at-end after test run: | mvn clean package -U -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -ntp --fail-at-end + source $TRAVIS_DIR/install-rocksdb.sh - name: Prepare env and service run: | diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh index 753eb45acc..04a472c1c3 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh @@ -17,29 +17,37 @@ # set -Eeuo pipefail +# Save original IFS to avoid leaking into parent shell when sourced +ORIG_IFS="${IFS}" IFS=$'\n\t' # Unified error capture for easy positioning trap 'echo "[install-rocksdb] error at line ${LINENO}: ${BASH_COMMAND}" >&2' ERR VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) SERVER_DIR="$(pwd)/hugegraph-server/apache-hugegraph-server-incubating-$VERSION" -BIN="$SERVER_DIR/bin" -LIB="$SERVER_DIR/lib" -DEST_DIR="$SERVER_DIR/library" +SERVER_BIN="$SERVER_DIR/bin" +SERVER_LIB="$SERVER_DIR/lib" +INSTALL_DEST_DIR="$SERVER_DIR/library" if [ ! -d "$SERVER_DIR" ]; then echo "Error: SERVER_DIR not found: $SERVER_DIR" >&2 exit 1 fi -if [ ! -d "$LIB" ]; then - echo "Error: LIB dir not found: $LIB" >&2 +if [ ! -d "$SERVER_LIB" ]; then + echo "Error: SERVER_LIB dir not found: $SERVER_LIB" >&2 exit 1 fi -if [ ! -f "$BIN/common-topling.sh" ]; then - echo "Error: common-topling.sh not found under: $BIN" >&2 +if [ ! -f "$SERVER_BIN/common-topling.sh" ]; then + echo "Error: common-topling.sh not found under: $SERVER_BIN" >&2 exit 1 fi -source "$BIN/common-topling.sh" +source "$SERVER_BIN/common-topling.sh" type preload_toplingdb >/dev/null 2>&1 || { echo "Error: function preload_toplingdb not found" >&2; exit 1; } -preload_toplingdb "$LIB" "$DEST_DIR" +preload_toplingdb "$SERVER_LIB" "$INSTALL_DEST_DIR" + +# Reset shell options to prevent affecting the parent shell when sourced +set +Eeuo pipefail +trap - ERR +# Restore original IFS +IFS="$ORIG_IFS" From 1b1e2fb54341cdb6155f58a57989358d069fcfd6 Mon Sep 17 00:00:00 2001 From: gydeng Date: Tue, 4 Nov 2025 10:43:27 +0000 Subject: [PATCH 41/51] chore(store): update license --- install-dist/release-docs/LICENSE | 7 ++----- install-dist/scripts/dependency/known-dependencies.txt | 6 ++---- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/install-dist/release-docs/LICENSE b/install-dist/release-docs/LICENSE index 9bb387714d..c97fc79491 100644 --- a/install-dist/release-docs/LICENSE +++ b/install-dist/release-docs/LICENSE @@ -439,9 +439,8 @@ The text of each license is also included in licenses/LICENSE-[project].txt. https://central.sonatype.com/artifact/net.java.dev.jna/jna/5.7.0 -> Apache 2.0 https://central.sonatype.com/artifact/com.github.jnr/jnr-ffi/2.1.7 -> Apache 2.0 https://central.sonatype.com/artifact/joda-time/joda-time/2.10.8 -> Apache 2.0 - https://central.sonatype.com/artifact/com.alipay.sofa/jraft-core/1.3.11 -> Apache 2.0 - https://central.sonatype.com/artifact/com.alipay.sofa/jraft-core/1.3.13 -> Apache 2.0 https://central.sonatype.com/artifact/com.alipay.sofa/jraft-core/1.3.9 -> Apache 2.0 + https://central.sonatype.com/artifact/com.alipay.sofa/jraft-core/1.3.14 -> Apache 2.0 https://central.sonatype.com/artifact/org.skyscreamer/jsonassert/1.5.0 -> Apache 2.0 https://central.sonatype.com/artifact/com.jayway.jsonpath/json-path/2.5.0 -> Apache 2.0 https://central.sonatype.com/artifact/com.googlecode.json-simple/json-simple/1.1 -> Apache 2.0 @@ -570,9 +569,7 @@ The text of each license is also included in licenses/LICENSE-[project].txt. https://central.sonatype.com/artifact/com.addthis.metrics/reporter-config3/3.0.3 -> Apache 2.0 https://central.sonatype.com/artifact/com.addthis.metrics/reporter-config-base/3.0.3 -> Apache 2.0 https://central.sonatype.com/artifact/org.opencypher/rewriting-9.0/9.0.20190305 -> Apache 2.0 - https://central.sonatype.com/artifact/org.rocksdb/rocksdbjni/6.29.5 -> Apache 2.0 - https://central.sonatype.com/artifact/org.rocksdb/rocksdbjni/7.2.2 -> Apache 2.0 - https://central.sonatype.com/artifact/org.rocksdb/rocksdbjni/7.7.3 -> Apache 2.0 + https://central.sonatype.com/artifact/org.rocksdb/rocksdbjni/8.10.2 -> Apache 2.0 https://github.com/hugegraph/toplingdb/packages/2550860 -> Apache 2.0 https://central.sonatype.com/artifact/org.fusesource/sigar/1.6.4 -> Apache 2.0 https://central.sonatype.com/artifact/io.prometheus/simpleclient/0.10.0 -> Apache 2.0 diff --git a/install-dist/scripts/dependency/known-dependencies.txt b/install-dist/scripts/dependency/known-dependencies.txt index f25b7b0308..4f899e9ad6 100644 --- a/install-dist/scripts/dependency/known-dependencies.txt +++ b/install-dist/scripts/dependency/known-dependencies.txt @@ -295,8 +295,7 @@ jna-5.7.0.jar jnr-ffi-2.1.7.jar jnr-x86asm-1.0.2.jar joda-time-2.10.8.jar -jraft-core-1.3.11.jar -jraft-core-1.3.13.jar +jraft-core-1.3.14.jar jraft-core-1.3.9.jar json-path-2.5.0.jar json-simple-1.1.jar @@ -482,8 +481,7 @@ psjava-0.1.19.jar reporter-config-base-3.0.3.jar reporter-config3-3.0.3.jar rewriting-9.0-9.0.20190305.jar -rocksdbjni-6.29.5.jar -rocksdbjni-7.7.3.jar +rocksdbjni-8.10.2.jar rocksdbjni-8.10.2-SNAPSHOT.jar scala-java8-compat_2.12-0.8.0.jar scala-library-2.12.7.jar From 2ff7be285335002864694e1528d79c9150f7af8b Mon Sep 17 00:00:00 2001 From: gydeng Date: Wed, 5 Nov 2025 12:02:54 +0000 Subject: [PATCH 42/51] fix(rocksdb-provider): remove the verification of the data path in the provider --- hugegraph-rocksdb-provider/pom.xml | 26 ++++-- .../provider/AbstractRocksDBProvider.java | 25 ------ .../provider/ToplingRocksDBProvider.java | 86 ++++++++++++------- hugegraph-server/hugegraph-core/pom.xml | 2 +- .../backend/store/rocksdb/RocksDBOptions.java | 77 ----------------- .../scripts/dependency/known-dependencies.txt | 1 - 6 files changed, 76 insertions(+), 141 deletions(-) diff --git a/hugegraph-rocksdb-provider/pom.xml b/hugegraph-rocksdb-provider/pom.xml index fc7be341cf..6f44d381b4 100644 --- a/hugegraph-rocksdb-provider/pom.xml +++ b/hugegraph-rocksdb-provider/pom.xml @@ -18,21 +18,24 @@ - - hugegraph - org.apache.hugegraph - ${revision} - ../pom.xml - 4.0.0 hugegraph-rocksdb-provider + ${revision} + ${project.artifactId} HugeGraph RocksDB Provider - A common adapter module for RocksDB and ToplingDB integration using SPI (Service Provider Interface) and Strategy Pattern for loose coupling and plugin-based architecture. + + hugegraph + org.apache.hugegraph + ${revision} + ../pom.xml + + @@ -52,14 +55,21 @@ org.slf4j slf4j-api - 1.7.5 + 2.0.9 + + + + + org.yaml + snakeyaml + 2.2 net.minidev json-smart - 2.4.8 + 2.3 diff --git a/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/AbstractRocksDBProvider.java b/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/AbstractRocksDBProvider.java index 515d3e66f6..b605b519c2 100644 --- a/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/AbstractRocksDBProvider.java +++ b/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/AbstractRocksDBProvider.java @@ -21,7 +21,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; import java.util.List; /** @@ -37,9 +36,6 @@ public abstract class AbstractRocksDBProvider implements RocksDBProvider { public final RocksDB openRocksDB(Options options, String dataPath) throws RocksDBException { LOG.debug("Opening RocksDB with provider: {} at path: {}", getProviderName(), dataPath); - // Ensure directory exists - ensureDirectoryExists(dataPath); - // Initialize provider if needed initialize(); @@ -59,9 +55,6 @@ public final RocksDB openRocksDB(Options options, String dataPath) throws RocksD public final RocksDB openRocksDB(DBOptions dbOptions, String dataPath, List cfDescriptors, List cfHandles) throws RocksDBException { - // Ensure directory exists - ensureDirectoryExists(dataPath); - // Initialize provider if needed initialize(); @@ -85,10 +78,6 @@ public final RocksDB openRocksDB(DBOptions dbOptions, String dataPath, LOG.debug("Opening RocksDB with extended parameters using provider: {} at path: {}", getProviderName(), dataPath); - - // Ensure directory exists - ensureDirectoryExists(dataPath); - // Initialize provider if needed initialize(); @@ -112,20 +101,6 @@ public void closeRocksDB(RocksDB rocksDB) { } } - /** - * Ensure database directory exists - */ - protected void ensureDirectoryExists(String dbPath) { - File dbDir = new File(dbPath); - if (!dbDir.exists()) { - boolean created = dbDir.mkdirs(); - if (!created) { - throw new RuntimeException("Failed to create database directory: " + dbPath); - } - LOG.info("Created database directory: {}", dbPath); - } - } - // ========== Abstract methods for provider-specific implementations ========== /** diff --git a/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/ToplingRocksDBProvider.java b/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/ToplingRocksDBProvider.java index c80ac4c7e7..8fe43e6a80 100644 --- a/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/ToplingRocksDBProvider.java +++ b/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/ToplingRocksDBProvider.java @@ -25,13 +25,20 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Locale; import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.Map; +import java.util.regex.Pattern; import net.minidev.json.JSONObject; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.SafeConstructor; +import org.yaml.snakeyaml.LoaderOptions; + /** * ToplingRocksDBProvider provides ToplingDB-specific RocksDB functionality. * This provider supports advanced ToplingDB features including: @@ -47,6 +54,12 @@ public class ToplingRocksDBProvider extends AbstractRocksDBProvider { private static final int PROVIDER_PRIORITY = 200; // Higher priority than standard private static final String SIDE_PLUGIN_REPO_CLASS = "org.rocksdb.SidePluginRepo"; + // Validation constants migrated from RocksDBOptions + private static final Pattern SAFE_PATH_PATTERN = + Pattern.compile("^[a-zA-Z0-9/_.-]+\\.yaml$"); + private static final String ALLOWED_CONFIG_DIR = "./conf/graphs/"; + private static final long MAX_CONFIG_FILE_SIZE = 1024 * 1024 * 10; // 10 MB + // Store repo objects for proper cleanup private final Map rocksDBToRepoMap = new ConcurrentHashMap<>(); @@ -314,9 +327,10 @@ private RocksDB validateAndStoreResult(Object result, Object repo, String dataPa * Utility function to convert options to JSON string for ToplingDB * Moved from RocksDBStdSessions */ - private static String converseOptionsToJsonString(String dataPath, List cfs) { + private static String converseOptionsToJsonString(String dataPath, List cfs) + throws RocksDBException { if (dataPath == null || dataPath.trim().isEmpty()) { - throw new IllegalArgumentException("dataPath cannot be null or empty"); + throw new RocksDBException("RocksDB dataPath cannot be null or empty"); } // sanitize path to avoid trailing slash causing empty namepart in native side String sanitizedPath = sanitizePath(dataPath); @@ -407,24 +421,43 @@ private static boolean validateConfiguration(String optionPath) { * This method will be available when hugegraph-rocksdb is in classpath. */ private static void validateOptionPath(String optionPath) { + // 1. Path format validation + if (optionPath == null || optionPath.isBlank()) { + throw new IllegalArgumentException("option_path can't be null or empty"); + } + if (!SAFE_PATH_PATTERN.matcher(optionPath).matches() || + optionPath.contains("..") || optionPath.contains("://")) { + throw new IllegalArgumentException("Invalid option_path format: " + optionPath); + } + String lower = optionPath.toLowerCase(Locale.ROOT); + if (!(lower.endsWith(".yaml") || lower.endsWith(".yml"))) { + throw new IllegalArgumentException("option_path must end with .yaml or .yml"); + } + + // 2. Normalize path and constrain under allowed directory + Path allowedDir = Paths.get(ALLOWED_CONFIG_DIR).toAbsolutePath().normalize(); + Path configPath = Paths.get(optionPath).toAbsolutePath().normalize(); + + if (!configPath.startsWith(allowedDir)) { + throw new SecurityException("option_path must be under " + ALLOWED_CONFIG_DIR); + } + + // 3. Validate file existence and readability + if (!Files.isRegularFile(configPath) || !Files.isReadable(configPath)) { + throw new IllegalArgumentException( + "Config file not found or not readable: " + configPath); + } + + // 4. File size limit (prevent DoS) + final long fileSize; try { - // Use reflection to call RocksDBOptions.validateOptionPath - Class rocksDBOptionsClass = - Class.forName("org.apache.hugegraph.backend.store.rocksdb.RocksDBOptions"); - Method validateMethod = - rocksDBOptionsClass.getMethod("validateOptionPath", String.class); - validateMethod.invoke(null, optionPath); + fileSize = Files.size(configPath); } catch (Exception e) { - // Fallback to basic validation if RocksDBOptions is not available - LOG.warn("RocksDBOptions.validateOptionPath not available, using basic validation: {}", - e.getMessage()); - if (optionPath == null || optionPath.isBlank()) { - throw new IllegalArgumentException("option_path can't be null or empty"); - } - if (!optionPath.toLowerCase().endsWith(".yaml") && - !optionPath.toLowerCase().endsWith(".yml")) { - throw new IllegalArgumentException("option_path must end with .yaml or .yml"); - } + throw new IllegalArgumentException("Failed to access config file size: " + configPath, + e); + } + if (fileSize > MAX_CONFIG_FILE_SIZE) { + throw new IllegalArgumentException("Config file too large: " + fileSize + " bytes"); } } @@ -433,19 +466,14 @@ private static void validateOptionPath(String optionPath) { * This method will be available when hugegraph-rocksdb is in classpath. */ private static void validateYamlContent(String yamlContent) { + // Use a safe YAML parser and disable dangerous features + Yaml yaml = new Yaml(new SafeConstructor(new LoaderOptions())); try { - // Use reflection to call RocksDBOptions.validateYamlContent - Class rocksDBOptionsClass = - Class.forName("org.apache.hugegraph.backend.store.rocksdb.RocksDBOptions"); - Method validateMethod = - rocksDBOptionsClass.getMethod("validateYamlContent", String.class); - validateMethod.invoke(null, yamlContent); + yaml.load(yamlContent); + // TODO: validate config schema } catch (Exception e) { - // Fallback to basic validation if RocksDBOptions is not available - LOG.warn( - "RocksDBOptions.validateYamlContent not available, skipping YAML validation: " + - "{}", - e.getMessage()); + throw new IllegalArgumentException( + "Invalid YAML configuration", e); } } diff --git a/hugegraph-server/hugegraph-core/pom.xml b/hugegraph-server/hugegraph-core/pom.xml index 0b12f8b25f..eb0e8d69bb 100644 --- a/hugegraph-server/hugegraph-core/pom.xml +++ b/hugegraph-server/hugegraph-core/pom.xml @@ -29,7 +29,7 @@ ${basedir}/.. - 1.3.11 + 1.3.14 0.7.4 5.12.1 1.8.0 diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java index 33816f5653..6f54e2f439 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java @@ -34,16 +34,6 @@ import org.rocksdb.DataBlockIndexType; import org.rocksdb.IndexType; -import java.util.regex.Pattern; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Locale; - -import org.yaml.snakeyaml.Yaml; -import org.yaml.snakeyaml.constructor.SafeConstructor; - import com.google.common.collect.ImmutableList; public class RocksDBOptions extends OptionHolder { @@ -54,11 +44,6 @@ private RocksDBOptions() { private static volatile RocksDBOptions instance; - private static final Pattern SAFE_PATH_PATTERN = - Pattern.compile("^[a-zA-Z0-9/_.-]+\\.yaml$"); - private static final String ALLOWED_CONFIG_DIR = "./conf/graphs/"; - private static final long MAX_CONFIG_FILE_SIZE = 1024 * 1024 * 10; // 10 MB - public static synchronized RocksDBOptions instance() { if (instance == null) { instance = new RocksDBOptions(); @@ -67,68 +52,6 @@ public static synchronized RocksDBOptions instance() { return instance; } - /** - * Validate the option_path string for safety and availability. - * - Enforce format, normalize and prevent path traversal - * - Restrict to an allowed base directory - * - Ensure the file exists, is readable, and within size limits - */ - public static void validateOptionPath(String optionPath) { - // 1. Path format validation - if (optionPath == null || optionPath.isBlank()) { - throw new IllegalArgumentException("option_path can't be null or empty"); - } - if (!SAFE_PATH_PATTERN.matcher(optionPath).matches() || - optionPath.contains("..") || optionPath.contains("://")) { - throw new IllegalArgumentException("Invalid option_path format: " + optionPath); - } - String lower = optionPath.toLowerCase(Locale.ROOT); - if (!(lower.endsWith(".yaml") || lower.endsWith(".yml"))) { - throw new IllegalArgumentException("option_path must end with .yaml or .yml"); - } - - // 2. Normalize path and constrain under allowed directory - Path allowedDir = Paths.get(ALLOWED_CONFIG_DIR).toAbsolutePath().normalize(); - Path configPath = Paths.get(optionPath).toAbsolutePath().normalize(); - - if (!configPath.startsWith(allowedDir)) { - throw new SecurityException("option_path must be under " + ALLOWED_CONFIG_DIR); - } - - // 3. Validate file existence and readability - if (!Files.isRegularFile(configPath) || !Files.isReadable(configPath)) { - throw new IllegalArgumentException( - "Config file not found or not readable: " + configPath); - } - - // 4. File size limit (prevent DoS) - final long fileSize; - try { - fileSize = Files.size(configPath); - } catch (IOException e) { - throw new IllegalArgumentException("Failed to access config file size: " + configPath, - e); - } - if (fileSize > MAX_CONFIG_FILE_SIZE) { - throw new IllegalArgumentException("Config file too large: " + fileSize + " bytes"); - } - } - - /** - * - Validates parsed structure schema and limits depth & node counts - */ - public static void validateYamlContent(String yamlContent) { - // Use a safe YAML parser and disable dangerous features - Yaml yaml = new Yaml(new SafeConstructor()); - try { - yaml.load(yamlContent); - // TODO: validate config schema - } catch (Exception e) { - throw new IllegalArgumentException( - "Invalid YAML configuration", e); - } - } - // TODO: the entire align style is wrong, change it to 4 space later public static final ConfigOption DATA_PATH = new ConfigOption<>( diff --git a/install-dist/scripts/dependency/known-dependencies.txt b/install-dist/scripts/dependency/known-dependencies.txt index 4f899e9ad6..8b44dd70bf 100644 --- a/install-dist/scripts/dependency/known-dependencies.txt +++ b/install-dist/scripts/dependency/known-dependencies.txt @@ -481,7 +481,6 @@ psjava-0.1.19.jar reporter-config-base-3.0.3.jar reporter-config3-3.0.3.jar rewriting-9.0-9.0.20190305.jar -rocksdbjni-8.10.2.jar rocksdbjni-8.10.2-SNAPSHOT.jar scala-java8-compat_2.12-0.8.0.jar scala-library-2.12.7.jar From e28e425e0fde32429fc57598a64d4c31002e40ed Mon Sep 17 00:00:00 2001 From: gydeng Date: Wed, 5 Nov 2025 12:03:32 +0000 Subject: [PATCH 43/51] fix(pd): fix dependency issues in CI --- .github/workflows/pd-store-ci.yml | 18 +++++++++++------- .../assembly/static/bin/start-hugegraph-pd.sh | 8 +++----- .../static/bin/start-hugegraph-store.sh | 9 ++++----- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/.github/workflows/pd-store-ci.yml b/.github/workflows/pd-store-ci.yml index 30cd4558f5..ae6577b917 100644 --- a/.github/workflows/pd-store-ci.yml +++ b/.github/workflows/pd-store-ci.yml @@ -33,6 +33,10 @@ jobs: with: fetch-depth: 5 + - name: Install deps + run: | + ${TRAVIS_DIR}/install-deps.sh + - name: Use staged maven repo settings run: | cp $HOME/.m2/settings.xml /tmp/settings.xml || true @@ -89,6 +93,12 @@ jobs: cp $HOME/.m2/settings.xml /tmp/settings.xml mv -vf .github/configs/settings.xml $HOME/.m2/settings.xml + - name: Package + # todo remove --fail-at-end after test + run: | + mvn clean package -U -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -ntp --fail-at-end + source $TRAVIS_DIR/install-rocksdb.sh + - name: Run common test run: | mvn test -pl hugegraph-pd/hg-pd-test -am -P pd-common-test @@ -99,14 +109,8 @@ jobs: # The above tests do not require starting a PD instance. - - name: Package - # todo remove --fail-at-end after test - run: | - mvn clean package -U -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -ntp --fail-at-end - - name: Prepare env and service run: | - source $TRAVIS_DIR/install-rocksdb.sh $TRAVIS_DIR/start-pd.sh - name: Run client test @@ -162,10 +166,10 @@ jobs: # todo remove --fail-at-end after test run: | mvn clean package -U -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -ntp --fail-at-end + source $TRAVIS_DIR/install-rocksdb.sh - name: Prepare env and service run: | - source $TRAVIS_DIR/install-rocksdb.sh $TRAVIS_DIR/start-pd.sh $TRAVIS_DIR/start-store.sh diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh index c1696f9e37..7b5821130f 100644 --- a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh +++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh @@ -61,16 +61,14 @@ PID_FILE="$BIN/pid" PARENT_DIR="$(cd "$TOP"/../ && pwd)" SERVER_VERSION_DIR="${SERVER_VERSION_DIR:-$(find_hugegraph_server_dir "$PARENT_DIR")}" -if [ -z "$SERVER_VERSION_DIR" ] || [ ! -d "$SERVER_VERSION_DIR" ]; then - echo "Error: failed to locate ${PARENT_DIR}/apache-hugegraph-server* . Set SERVER_VERSION_DIR manually." >&2 - exit 1 -fi ensure_path_writable "$LOGS" ensure_path_writable "$PLUGINS" # preload rocksdb/toplingdb -source "$SERVER_VERSION_DIR/bin/preload-topling.sh" +if [ -e "$SERVER_VERSION_DIR/bin/preload-topling.sh" ]; then + source "$SERVER_VERSION_DIR/bin/preload-topling.sh" +fi # The maximum and minimum heap memory that service can use MAX_MEM=$((32 * 1024)) diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh index bbcbad6242..fc72e4c9b5 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh @@ -40,10 +40,6 @@ PID_FILE="$BIN/pid" . "$BIN"/util.sh PARENT_DIR="$(cd "$TOP"/../ && pwd)" SERVER_VERSION_DIR="${SERVER_VERSION_DIR:-$(find_hugegraph_server_dir "$PARENT_DIR")}" -if [ -z "$SERVER_VERSION_DIR" ] || [ ! -d "$SERVER_VERSION_DIR" ]; then - echo "Error: failed to locate ${PARENT_DIR}/apache-hugegraph-server* . Set SERVER_VERSION_DIR manually." >&2 - exit 1 -fi arch=$(uname -m) echo "Current arch: $arch" @@ -70,7 +66,10 @@ else echo "Unsupported architecture: $arch" fi -source "$SERVER_VERSION_DIR/bin/preload-topling.sh" +# preload rocksdb/toplingdb +if [ -e "$SERVER_VERSION_DIR/bin/preload-topling.sh" ]; then + source "$SERVER_VERSION_DIR/bin/preload-topling.sh" +fi ##pd/store max user processes, ulimit -u # Reduce the maximum number of processes that can be opened by a normal dev/user From 723fad5c4d3ef55647df8eebc24fd00c25fe7800 Mon Sep 17 00:00:00 2001 From: gydeng Date: Wed, 5 Nov 2025 12:37:42 +0000 Subject: [PATCH 44/51] fix(pd): add install deps in pd-store ci --- .github/workflows/pd-store-ci.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pd-store-ci.yml b/.github/workflows/pd-store-ci.yml index ae6577b917..ce7efdfa3b 100644 --- a/.github/workflows/pd-store-ci.yml +++ b/.github/workflows/pd-store-ci.yml @@ -33,10 +33,6 @@ jobs: with: fetch-depth: 5 - - name: Install deps - run: | - ${TRAVIS_DIR}/install-deps.sh - - name: Use staged maven repo settings run: | cp $HOME/.m2/settings.xml /tmp/settings.xml || true @@ -93,6 +89,10 @@ jobs: cp $HOME/.m2/settings.xml /tmp/settings.xml mv -vf .github/configs/settings.xml $HOME/.m2/settings.xml + - name: Install deps + run: | + ${TRAVIS_DIR}/install-deps.sh + - name: Package # todo remove --fail-at-end after test run: | @@ -162,6 +162,10 @@ jobs: cp $HOME/.m2/settings.xml /tmp/settings.xml mv -vf .github/configs/settings.xml $HOME/.m2/settings.xml + - name: Install deps + run: | + ${TRAVIS_DIR}/install-deps.sh + - name: Package # todo remove --fail-at-end after test run: | From 9f5fc8238e0c6622a7d6327c488180069224a414 Mon Sep 17 00:00:00 2001 From: gydeng Date: Thu, 6 Nov 2025 07:32:54 +0000 Subject: [PATCH 45/51] fix(pd): fix environment variable issues in ci --- .../src/assembly/static/bin/start-hugegraph-pd.sh | 6 +++--- .../src/assembly/static/bin/common-topling.sh | 11 +++++++++++ .../src/assembly/static/bin/start-hugegraph-store.sh | 6 +++--- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh index 7b5821130f..689ba910ee 100644 --- a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh +++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh @@ -60,14 +60,14 @@ PID_FILE="$BIN/pid" . "$BIN"/util.sh PARENT_DIR="$(cd "$TOP"/../ && pwd)" -SERVER_VERSION_DIR="${SERVER_VERSION_DIR:-$(find_hugegraph_server_dir "$PARENT_DIR")}" +SERVER_DIR="${SERVER_DIR:-$(find_hugegraph_server_dir "$PARENT_DIR")}" ensure_path_writable "$LOGS" ensure_path_writable "$PLUGINS" # preload rocksdb/toplingdb -if [ -e "$SERVER_VERSION_DIR/bin/preload-topling.sh" ]; then - source "$SERVER_VERSION_DIR/bin/preload-topling.sh" +if [ -e "$SERVER_DIR/bin/preload-topling.sh" ]; then + source "$SERVER_DIR/bin/preload-topling.sh" fi # The maximum and minimum heap memory that service can use diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh index c69b6e0c6e..6966752bff 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh @@ -249,6 +249,17 @@ function preload_toplingdb() { if [ -f "$dest_dir/librocksdbjni-linux64.so" ] && [[ ":${LD_PRELOAD:-}:" != *"librocksdbjni-linux64.so:"* ]]; then export LD_PRELOAD="${LD_PRELOAD:+$LD_PRELOAD:}$dest_dir/librocksdbjni-linux64.so" fi + + # Persist environment for subsequent GitHub Actions steps + # so LD_* variables survive across separate run blocks. + if [ -n "${GITHUB_ENV:-}" ] && [ -w "$GITHUB_ENV" ]; then + { + echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" + echo "LD_PRELOAD=$LD_PRELOAD" + echo "SERVER_DIR=$SERVER_DIR" + } >> "$GITHUB_ENV" || true + echo "[common-topling] Exported LD_LIBRARY_PATH and LD_PRELOAD to GITHUB_ENV" >&2 || true + fi else echo "Warn: LD paths skipped, directory '$dest_dir' does not exist." >&2 fi diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh index fc72e4c9b5..42f682b90c 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh @@ -39,7 +39,7 @@ PID_FILE="$BIN/pid" . "$BIN"/util.sh PARENT_DIR="$(cd "$TOP"/../ && pwd)" -SERVER_VERSION_DIR="${SERVER_VERSION_DIR:-$(find_hugegraph_server_dir "$PARENT_DIR")}" +SERVER_DIR="${SERVER_DIR:-$(find_hugegraph_server_dir "$PARENT_DIR")}" arch=$(uname -m) echo "Current arch: $arch" @@ -67,8 +67,8 @@ else fi # preload rocksdb/toplingdb -if [ -e "$SERVER_VERSION_DIR/bin/preload-topling.sh" ]; then - source "$SERVER_VERSION_DIR/bin/preload-topling.sh" +if [ -e "$SERVER_DIR/bin/preload-topling.sh" ]; then + source "$SERVER_DIR/bin/preload-topling.sh" fi ##pd/store max user processes, ulimit -u From f36b8ab646a422e1938383fed7332aac9100edf4 Mon Sep 17 00:00:00 2001 From: gydeng Date: Fri, 7 Nov 2025 07:08:46 +0000 Subject: [PATCH 46/51] fix(pd): fix the snakeyaml dependency issue between rocksdb-provider and pd --- hugegraph-pd/hg-pd-core/pom.xml | 7 ++++ .../assembly/static/bin/start-hugegraph-pd.sh | 6 ++-- .../src/assembly/static/conf}/rocksdb_pd.yaml | 2 +- .../provider/ToplingRocksDBProvider.java | 2 +- .../src/assembly/static/bin/common-topling.sh | 33 +++++++++++++++---- .../static/conf/graphs/rocksdb_server.yaml | 2 +- .../src/assembly/travis/install-rocksdb.sh | 12 +++---- .../static/bin/start-hugegraph-store.sh | 6 ++-- .../assembly/static/conf/application-pd.yml | 2 +- .../assembly/static/conf}/rocksdb_store.yaml | 2 +- 10 files changed, 50 insertions(+), 24 deletions(-) rename {hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs => hugegraph-pd/hg-pd-dist/src/assembly/static/conf}/rocksdb_pd.yaml (99%) rename {hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs => hugegraph-store/hg-store-dist/src/assembly/static/conf}/rocksdb_store.yaml (99%) diff --git a/hugegraph-pd/hg-pd-core/pom.xml b/hugegraph-pd/hg-pd-core/pom.xml index 03988d8a0d..c0689ac482 100644 --- a/hugegraph-pd/hg-pd-core/pom.xml +++ b/hugegraph-pd/hg-pd-core/pom.xml @@ -50,6 +50,13 @@ org.apache.hugegraph hugegraph-rocksdb-provider ${revision} + + + + org.yaml + snakeyaml + + org.apache.hugegraph diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh index 689ba910ee..7b5821130f 100644 --- a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh +++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh @@ -60,14 +60,14 @@ PID_FILE="$BIN/pid" . "$BIN"/util.sh PARENT_DIR="$(cd "$TOP"/../ && pwd)" -SERVER_DIR="${SERVER_DIR:-$(find_hugegraph_server_dir "$PARENT_DIR")}" +SERVER_VERSION_DIR="${SERVER_VERSION_DIR:-$(find_hugegraph_server_dir "$PARENT_DIR")}" ensure_path_writable "$LOGS" ensure_path_writable "$PLUGINS" # preload rocksdb/toplingdb -if [ -e "$SERVER_DIR/bin/preload-topling.sh" ]; then - source "$SERVER_DIR/bin/preload-topling.sh" +if [ -e "$SERVER_VERSION_DIR/bin/preload-topling.sh" ]; then + source "$SERVER_VERSION_DIR/bin/preload-topling.sh" fi # The maximum and minimum heap memory that service can use diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_pd.yaml b/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/rocksdb_pd.yaml similarity index 99% rename from hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_pd.yaml rename to hugegraph-pd/hg-pd-dist/src/assembly/static/conf/rocksdb_pd.yaml index 440c515056..a1a1601978 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_pd.yaml +++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/rocksdb_pd.yaml @@ -17,7 +17,7 @@ # common parameters http: # normally parent path of db path - document_root: ./library + document_root: /dev/shm/rocksdb_resource listening_ports: '127.0.0.1:2011' setenv: StrSimpleEnvNameNotOverwrite: StringValue diff --git a/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/ToplingRocksDBProvider.java b/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/ToplingRocksDBProvider.java index 8fe43e6a80..b7324949fb 100644 --- a/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/ToplingRocksDBProvider.java +++ b/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/ToplingRocksDBProvider.java @@ -57,7 +57,7 @@ public class ToplingRocksDBProvider extends AbstractRocksDBProvider { // Validation constants migrated from RocksDBOptions private static final Pattern SAFE_PATH_PATTERN = Pattern.compile("^[a-zA-Z0-9/_.-]+\\.yaml$"); - private static final String ALLOWED_CONFIG_DIR = "./conf/graphs/"; + private static final String ALLOWED_CONFIG_DIR = "./conf/"; private static final long MAX_CONFIG_FILE_SIZE = 1024 * 1024 * 10; // 10 MB // Store repo objects for proper cleanup diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh index 6966752bff..43004ba6d9 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh @@ -19,8 +19,6 @@ set -Eeuo pipefail IFS=$'\n\t' trap 'echo "[common-topling] error at line ${LINENO}: ${BASH_COMMAND}" >&2' ERR -BIN="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -TOP="$(cd "$BIN"/../ && pwd)" GITHUB="https://github.com" function abs_path() { @@ -74,6 +72,13 @@ function extract_html_css_from_jar() { local jar_file="$1" local dest_dir="$2" local abs_jar_path + # Prefer /dev/shm on Linux for speed; fallback to TMPDIR or /tmp + local resource_target + if [ "$(uname -s)" = "Linux" ] && [ -d /dev/shm ]; then + resource_target="/dev/shm/rocksdb_resource" + else + resource_target="${TMPDIR:-/tmp}/rocksdb_resource" + fi if [ ! -f "$jar_file" ]; then echo "Error: JAR file '$jar_file' does not exist." >&2 @@ -104,6 +109,18 @@ function extract_html_css_from_jar() { return $code fi } + + mkdir -p "$resource_target" || { + echo "Error: Cannot create target directory '$resource_target'." >&2 + return 1 + } + + if compgen -G "$dest_dir"/*.html >/dev/null 2>&1; then + cp -f "$dest_dir"/*.html "$resource_target"/ + fi + if compgen -G "$dest_dir"/*.css >/dev/null 2>&1; then + cp -f "$dest_dir"/*.css "$resource_target"/ + fi } function ensure_libaio_symlink() { @@ -164,7 +181,8 @@ function download_and_verify() { } function download_and_setup_jemalloc() { - local arch lib_file download_url expected_md5 system_lib + local arch lib_file download_url expected_md5 system_lib top + top=$1 # Prefer system-installed jemalloc if available # Try ldconfig first to locate the shared object @@ -204,11 +222,11 @@ function download_and_setup_jemalloc() { # System jemalloc not found, try to download the correct library for the architecture if [[ $arch == "aarch64" || $arch == "arm64" ]]; then - lib_file="$TOP/bin/libjemalloc_aarch64.so" + lib_file="$top/bin/libjemalloc_aarch64.so" download_url="${GITHUB}/apache/hugegraph-doc/raw/binary-1.5/dist/server/libjemalloc_aarch64.so" expected_md5="2a631d2f81837f9d5864586761c5e380" elif [[ $arch == "x86_64" ]]; then - lib_file="$TOP/bin/libjemalloc.so" + lib_file="$top/bin/libjemalloc.so" download_url="${GITHUB}/apache/hugegraph-doc/raw/binary-1.5/dist/server/libjemalloc.so" expected_md5="fd61765eec3bfea961b646c269f298df" else @@ -230,6 +248,7 @@ function download_and_setup_jemalloc() { function preload_toplingdb() { local lib_dir="$1" local dest_dir="$2" + local top="$(cd "$lib_dir"/../ && pwd)" local jar_file jar_file=$(ls -1 "$lib_dir"/rocksdbjni*.jar 2>/dev/null | sort -V | tail -n1 || true) @@ -239,7 +258,7 @@ function preload_toplingdb() { fi ensure_libaio_symlink - download_and_setup_jemalloc + download_and_setup_jemalloc "$top" extract_so_with_jar "$jar_file" "$dest_dir" if [ -d "$dest_dir" ]; then if [[ ":${LD_LIBRARY_PATH:-}:" != *":$dest_dir:"* ]]; then @@ -256,7 +275,7 @@ function preload_toplingdb() { { echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" echo "LD_PRELOAD=$LD_PRELOAD" - echo "SERVER_DIR=$SERVER_DIR" + echo "SERVER_VERSION_DIR=$SERVER_VERSION_DIR" } >> "$GITHUB_ENV" || true echo "[common-topling] Exported LD_LIBRARY_PATH and LD_PRELOAD to GITHUB_ENV" >&2 || true fi diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_server.yaml b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_server.yaml index 440c515056..a1a1601978 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_server.yaml +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_server.yaml @@ -17,7 +17,7 @@ # common parameters http: # normally parent path of db path - document_root: ./library + document_root: /dev/shm/rocksdb_resource listening_ports: '127.0.0.1:2011' setenv: StrSimpleEnvNameNotOverwrite: StringValue diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh index 04a472c1c3..05f1085ffe 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh @@ -24,13 +24,13 @@ IFS=$'\n\t' trap 'echo "[install-rocksdb] error at line ${LINENO}: ${BASH_COMMAND}" >&2' ERR VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) -SERVER_DIR="$(pwd)/hugegraph-server/apache-hugegraph-server-incubating-$VERSION" -SERVER_BIN="$SERVER_DIR/bin" -SERVER_LIB="$SERVER_DIR/lib" -INSTALL_DEST_DIR="$SERVER_DIR/library" +SERVER_VERSION_DIR="$(pwd)/hugegraph-server/apache-hugegraph-server-incubating-$VERSION" +SERVER_BIN="$SERVER_VERSION_DIR/bin" +SERVER_LIB="$SERVER_VERSION_DIR/lib" +INSTALL_DEST_DIR="$SERVER_VERSION_DIR/library" -if [ ! -d "$SERVER_DIR" ]; then - echo "Error: SERVER_DIR not found: $SERVER_DIR" >&2 +if [ ! -d "$SERVER_VERSION_DIR" ]; then + echo "Error: SERVER_VERSION_DIR not found: $SERVER_VERSION_DIR" >&2 exit 1 fi if [ ! -d "$SERVER_LIB" ]; then diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh index 42f682b90c..fc72e4c9b5 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh @@ -39,7 +39,7 @@ PID_FILE="$BIN/pid" . "$BIN"/util.sh PARENT_DIR="$(cd "$TOP"/../ && pwd)" -SERVER_DIR="${SERVER_DIR:-$(find_hugegraph_server_dir "$PARENT_DIR")}" +SERVER_VERSION_DIR="${SERVER_VERSION_DIR:-$(find_hugegraph_server_dir "$PARENT_DIR")}" arch=$(uname -m) echo "Current arch: $arch" @@ -67,8 +67,8 @@ else fi # preload rocksdb/toplingdb -if [ -e "$SERVER_DIR/bin/preload-topling.sh" ]; then - source "$SERVER_DIR/bin/preload-topling.sh" +if [ -e "$SERVER_VERSION_DIR/bin/preload-topling.sh" ]; then + source "$SERVER_VERSION_DIR/bin/preload-topling.sh" fi ##pd/store max user processes, ulimit -u diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/conf/application-pd.yml b/hugegraph-store/hg-store-dist/src/assembly/static/conf/application-pd.yml index 3788e0a89f..737617e0d9 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/conf/application-pd.yml +++ b/hugegraph-store/hg-store-dist/src/assembly/static/conf/application-pd.yml @@ -33,6 +33,6 @@ rocksdb: # For each rocksdb, the number of memtables reaches this value for writing to disk. min_write_buffer_number_to_merge: 16 # Configuration file path for rocksdb/toplingdb - # option_path: ./conf/graphs/rocksdb_store.yaml + # option_path: ./conf/rocksdb_store.yaml # Whether to start Topling's HTTP service # open_http: false diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_store.yaml b/hugegraph-store/hg-store-dist/src/assembly/static/conf/rocksdb_store.yaml similarity index 99% rename from hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_store.yaml rename to hugegraph-store/hg-store-dist/src/assembly/static/conf/rocksdb_store.yaml index 440c515056..a1a1601978 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/rocksdb_store.yaml +++ b/hugegraph-store/hg-store-dist/src/assembly/static/conf/rocksdb_store.yaml @@ -17,7 +17,7 @@ # common parameters http: # normally parent path of db path - document_root: ./library + document_root: /dev/shm/rocksdb_resource listening_ports: '127.0.0.1:2011' setenv: StrSimpleEnvNameNotOverwrite: StringValue From 235d50c809cca689cda31ebe6b3750f09001fa71 Mon Sep 17 00:00:00 2001 From: gydeng Date: Sat, 8 Nov 2025 05:33:10 +0000 Subject: [PATCH 47/51] chore(rocksdb-provider): adopt the recommendations from ai review --- .github/workflows/pd-store-ci.yml | 12 +-- .../assembly/static/bin/start-hugegraph-pd.sh | 2 +- hugegraph-rocksdb-provider/pom.xml | 2 +- .../provider/ToplingRocksDBProvider.java | 98 ++++++++++++++++--- .../static/bin/start-hugegraph-store.sh | 2 +- 5 files changed, 94 insertions(+), 22 deletions(-) diff --git a/.github/workflows/pd-store-ci.yml b/.github/workflows/pd-store-ci.yml index ce7efdfa3b..c04ec749ac 100644 --- a/.github/workflows/pd-store-ci.yml +++ b/.github/workflows/pd-store-ci.yml @@ -91,13 +91,13 @@ jobs: - name: Install deps run: | - ${TRAVIS_DIR}/install-deps.sh + "${TRAVIS_DIR}/install-deps.sh" || exit 1 - name: Package # todo remove --fail-at-end after test run: | mvn clean package -U -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -ntp --fail-at-end - source $TRAVIS_DIR/install-rocksdb.sh + source "${TRAVIS_DIR}/install-rocksdb.sh" - name: Run common test run: | @@ -164,13 +164,13 @@ jobs: - name: Install deps run: | - ${TRAVIS_DIR}/install-deps.sh + "${TRAVIS_DIR}/install-deps.sh" || exit 1 - name: Package # todo remove --fail-at-end after test run: | mvn clean package -U -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -ntp --fail-at-end - source $TRAVIS_DIR/install-rocksdb.sh + source "${TRAVIS_DIR}/install-rocksdb.sh" - name: Prepare env and service run: | @@ -239,7 +239,7 @@ jobs: - name: Install deps run: | - ${TRAVIS_DIR}/install-deps.sh + "${TRAVIS_DIR}/install-deps.sh" || exit 1 - name: use staged maven repo settings if: ${{ env.USE_STAGE == 'true' }} @@ -251,7 +251,7 @@ jobs: # todo remove --fail-at-end after test run: | mvn clean package -U -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -ntp --fail-at-end - source $TRAVIS_DIR/install-rocksdb.sh + source "${TRAVIS_DIR}/install-rocksdb.sh" - name: Prepare env and service run: | diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh index 7b5821130f..dc88c1d897 100644 --- a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh +++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh @@ -66,7 +66,7 @@ ensure_path_writable "$LOGS" ensure_path_writable "$PLUGINS" # preload rocksdb/toplingdb -if [ -e "$SERVER_VERSION_DIR/bin/preload-topling.sh" ]; then +if [ -n "$SERVER_VERSION_DIR" ] && [ -e "$SERVER_VERSION_DIR/bin/preload-topling.sh" ]; then source "$SERVER_VERSION_DIR/bin/preload-topling.sh" fi diff --git a/hugegraph-rocksdb-provider/pom.xml b/hugegraph-rocksdb-provider/pom.xml index 6f44d381b4..8461b13490 100644 --- a/hugegraph-rocksdb-provider/pom.xml +++ b/hugegraph-rocksdb-provider/pom.xml @@ -76,7 +76,7 @@ junit junit - 4.13.1 + 4.13.2 test diff --git a/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/ToplingRocksDBProvider.java b/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/ToplingRocksDBProvider.java index b7324949fb..1bdb6ecc14 100644 --- a/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/ToplingRocksDBProvider.java +++ b/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/ToplingRocksDBProvider.java @@ -100,6 +100,7 @@ protected RocksDB doOpenRocksDB(Options options, String dataPath) throws RocksDB @Override public RocksDB openRocksDB(Options options, String dataPath, String optionPath, Boolean openHttp) throws RocksDBException { + initialize(); return doOpenRocksDB(options, dataPath, optionPath, openHttp); } @@ -158,30 +159,37 @@ protected RocksDB doOpenRocksDB(DBOptions dbOptions, String dataPath, private RocksDB openWithToplingFeatures(Options options, String dataPath, String optionPath, Boolean openHttp) throws RocksDBException { + Object repo = null; + RocksDB opened = null; + boolean registered = false; try { // Initialize ToplingDB repo with common operations - Object repo = initializeToplingRepo(options, dataPath, optionPath); + repo = initializeToplingRepo(options, dataPath, optionPath); // Open database with default column families Class sidePluginRepoClass = repo.getClass(); Method openDBMethod = sidePluginRepoClass.getMethod("openDB", String.class); Object result = openDBMethod.invoke(repo, converseOptionsToJsonString(dataPath, null)); + // Validate and store result before starting HTTP server + opened = validateAndStoreResult(result, repo, dataPath, 0); + registered = true; + // Start HTTP server if needed startHttpServerIfNeeded(repo, dataPath, openHttp, optionPath); - // Validate and store result - return validateAndStoreResult(result, repo, dataPath, 0); + return opened; } catch (InvocationTargetException e) { + cleanupFailedOpen(opened, repo, registered, null); Throwable cause = e.getCause(); if (cause instanceof RocksDBException) { throw (RocksDBException) cause; - } else { - throw new RocksDBException( - "Failed to open DB with SidePluginRepo: " + cause.getMessage()); } + throw new RocksDBException( + "Failed to open DB with SidePluginRepo: " + cause.getMessage()); } catch (Exception e) { + cleanupFailedOpen(opened, repo, registered, null); throw new RocksDBException("Failed to open ToplingDB: " + e.getMessage()); } } @@ -194,10 +202,12 @@ private RocksDB openWithToplingFeaturesAndCF(DBOptions dbOptions, String dataPat List cfHandles, String optionPath, Boolean openHttp) throws RocksDBException { - + Object repo = null; + RocksDB opened = null; + boolean registered = false; try { // Initialize ToplingDB repo with common operations - Object repo = initializeToplingRepo(dbOptions, dataPath, optionPath); + repo = initializeToplingRepo(dbOptions, dataPath, optionPath); // Prepare column family names for JSON List cfNames = new java.util.ArrayList<>(); @@ -212,21 +222,25 @@ private RocksDB openWithToplingFeaturesAndCF(DBOptions dbOptions, String dataPat converseOptionsToJsonString(dataPath, cfNames), cfHandles); + // Validate and store result before starting HTTP server + opened = validateAndStoreResult(result, repo, dataPath, cfDescriptors.size()); + registered = true; + // Start HTTP server if needed startHttpServerIfNeeded(repo, dataPath, openHttp, optionPath); - // Validate and store result - return validateAndStoreResult(result, repo, dataPath, cfDescriptors.size()); + return opened; } catch (InvocationTargetException e) { + cleanupFailedOpen(opened, repo, registered, cfHandles); Throwable cause = e.getCause(); if (cause instanceof RocksDBException) { throw (RocksDBException) cause; - } else { - throw new RocksDBException( - "Failed to open DB with SidePluginRepo: " + cause.getMessage()); } + throw new RocksDBException( + "Failed to open DB with SidePluginRepo: " + cause.getMessage()); } catch (Exception e) { + cleanupFailedOpen(opened, repo, registered, cfHandles); LOG.error("Failed to open ToplingDB with column families", e); throw new RocksDBException("Failed to open ToplingDB: " + e.getMessage()); } @@ -323,6 +337,64 @@ private RocksDB validateAndStoreResult(Object result, Object repo, String dataPa } } + /** + * Cleanup resources when opening RocksDB fails after successful openDB call. + * This prevents resource leaks when HTTP server startup or other post-open operations fail. + * + * @param opened The RocksDB instance that was opened (may be null) + * @param repo The SidePluginRepo instance (may be null) + * @param registered Whether the RocksDB was registered in rocksDBToRepoMap + * @param cfHandles List of column family handles to close (may be null) + */ + private void cleanupFailedOpen(RocksDB opened, Object repo, boolean registered, + List cfHandles) { + if (opened == null && repo == null) { + // Nothing to clean up + return; + } + + LOG.warn("Cleaning up resources after failed RocksDB open operation"); + + // Remove from map if registered + if (registered && opened != null) { + rocksDBToRepoMap.remove(opened); + } + + // Close column family handles if provided + if (cfHandles != null && !cfHandles.isEmpty()) { + for (ColumnFamilyHandle cfHandle : cfHandles) { + if (cfHandle != null) { + try { + cfHandle.close(); + } catch (Exception e) { + LOG.warn("Failed to close column family handle during cleanup", e); + } + } + } + } + + // Close RocksDB instance + if (opened != null) { + try { + opened.close(); + } catch (Exception e) { + LOG.warn("Failed to close RocksDB instance during cleanup", e); + } + } + + // Close SidePluginRepo + if (repo != null) { + try { + Class sidePluginRepoClass = repo.getClass(); + Method closeAllDBMethod = sidePluginRepoClass.getMethod("closeAllDB"); + closeAllDBMethod.invoke(repo); + LOG.debug("Successfully called closeAllDB() on SidePluginRepo during cleanup"); + } catch (Exception e) { + LOG.warn("Failed to call closeAllDB() on SidePluginRepo during cleanup", e); + } + } + } + /** * Utility function to convert options to JSON string for ToplingDB * Moved from RocksDBStdSessions diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh index fc72e4c9b5..ec33636364 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh @@ -67,7 +67,7 @@ else fi # preload rocksdb/toplingdb -if [ -e "$SERVER_VERSION_DIR/bin/preload-topling.sh" ]; then +if [ -n "$SERVER_VERSION_DIR" ] && [ -e "$SERVER_VERSION_DIR/bin/preload-topling.sh" ]; then source "$SERVER_VERSION_DIR/bin/preload-topling.sh" fi From 9c154ee419dc8d0f82f514b0eb09301fbfa81d54 Mon Sep 17 00:00:00 2001 From: ken <2979602290@qq.com> Date: Thu, 9 Jul 2026 10:36:08 +0800 Subject: [PATCH 48/51] feat(rocksdb): change provider selection from implicit classpath auto-detection to explicit `rocksdb.provider` configuration --- ...xplicit-config-independent-provider-sop.md | 420 ++++++++++++++++++ .../apache/hugegraph/pd/config/PDConfig.java | 2 + .../hugegraph/pd/store/HgKVStoreImpl.java | 2 + hugegraph-rocksdb-provider/pom.xml | 8 +- .../rocksdb/provider/RocksDBProvider.java | 7 - .../provider/RocksDBProviderLoader.java | 216 ++++----- .../provider/StandardRocksDBProvider.java | 6 - .../provider/ToplingRocksDBProvider.java | 6 - .../provider/RocksDBProviderLoaderTest.java | 170 +++++++ .../provider/StandardRocksDBProviderTest.java | 189 ++++++++ .../provider/ToplingRocksDBProviderTest.java | 92 ++++ .../backend/store/rocksdb/RocksDBOptions.java | 9 + .../store/rocksdb/RocksDBStdSessions.java | 8 + .../rocksdb/access/RocksDBOptions.java | 8 + .../rocksdb/access/RocksDBSession.java | 3 + pom.xml | 14 + 16 files changed, 1000 insertions(+), 160 deletions(-) create mode 100644 docs/toplingdb/toplingdb-explicit-config-independent-provider-sop.md create mode 100644 hugegraph-rocksdb-provider/src/test/java/org/apache/hugegraph/rocksdb/provider/RocksDBProviderLoaderTest.java create mode 100644 hugegraph-rocksdb-provider/src/test/java/org/apache/hugegraph/rocksdb/provider/StandardRocksDBProviderTest.java create mode 100644 hugegraph-rocksdb-provider/src/test/java/org/apache/hugegraph/rocksdb/provider/ToplingRocksDBProviderTest.java diff --git a/docs/toplingdb/toplingdb-explicit-config-independent-provider-sop.md b/docs/toplingdb/toplingdb-explicit-config-independent-provider-sop.md new file mode 100644 index 0000000000..e563479da2 --- /dev/null +++ b/docs/toplingdb/toplingdb-explicit-config-independent-provider-sop.md @@ -0,0 +1,420 @@ +# ToplingDB 显式配置 + 独立 Provider 重构 SOP + +## 一、背景与问题 + +### 1.1 现状 + +当前 ToplingDB 集成方案存在以下核心问题: + +| 问题 | 具体表现 | +|------|----------| +| **隐式激活** | `ToplingRocksDBProvider` 通过 `Class.forName("org.rocksdb.SidePluginRepo")` 自动探测 classpath,只要 ToplingDB JAR 存在就自动接管所有 RocksDB 操作(priority=200 > standard=100),用户无法显式选择 | +| **共享坐标污染** | ToplingDB 与标准 RocksDB 共用 `org.rocksdb:rocksdbjni` 坐标(仅版本/来源不同),导致 Maven 依赖仲裁不可控 | +| **影响纯 RocksDB 用户** | 不需要 ToplingDB 的用户也被 SNAPSHOT 依赖、GitHub Packages 仓库配置所困扰 | + +### 1.2 目标 + +1. **显式配置**:用户通过 `rocksdb.provider=standard|topling` 明确选择引擎,不再依赖 classpath 自动探测 +2. **部署时 Drop-in**:运维不需要 Maven 配置,下载 ToplingDB addon 包放入部署目录即可 +3. **模块结构不变**:保留现有 `hugegraph-rocksdb-provider` 单模块,内部简化逻辑 + +--- + +## 二、分发模型 + +### 2.1 构建与部署分离 + +ToplingDB 本身就需要下载 native `.so` 库和可能的外置二进制文件,因此对终端用户来说是一个 **部署时 addon**,而非编译时依赖。 + +``` +┌──────────────────────────────────────────────────────────┐ +│ 默认构建产物 │ +│ mvn clean package │ +│ ├── lib/rocksdbjni-8.10.2.jar (标准 RocksDB) │ +│ └── lib/hugegraph-rocksdb-provider.jar │ +└──────────────────────────────────────────────────────────┘ + + ↓ 运维选择启用 ToplingDB ↓ + +┌──────────────────────────────────────────────────────────┐ +│ ToplingDB Addon 包 (独立下载) │ +│ toplingdb-addon-1.0.0-linux-x86_64.tar.gz │ +│ ├── lib/toplingdb-jni-1.0.0.jar (替换 rocksdbjni) │ +│ ├── native/librocksdbjni-linux64.so │ +│ ├── native/libterark-*.so (可选) │ +│ ├── web/ (ToplingDB HTTP 监控 UI) │ +│ └── conf/rocksdb_plus.yaml.template │ +└──────────────────────────────────────────────────────────┘ +``` + +### 2.2 两种角色 + +| 角色 | ToplingDB 怎么来 | 需要 Maven 配置? | +|------|-----------------|-----------------| +| **运维/部署者** | 下载 addon 包,解压到部署目录,改配置 | 不需要 | +| **开发者** | POM 中 `-P toplingdb` 引入(仅编译/调试用) | 需要 | + +### 2.3 部署流程(运维视角) + +```bash +# 1. 下载 ToplingDB addon 包 +wget https://github.com/hugegraph/toplingdb/releases/download/v1.0.0/toplingdb-addon-1.0.0-linux-x86_64.tar.gz + +# 2. 解压到 HugeGraph 安装目录 +tar -xzf toplingdb-addon-1.0.0-linux-x86_64.tar.gz -C /opt/hugegraph/ + +# 3. 替换标准 RocksDB JAR(互斥) +rm /opt/hugegraph/lib/rocksdbjni-*.jar + +# 4. 修改配置 +vi conf/hugegraph.properties +# rocksdb.provider=topling +# rocksdb.option_path=./conf/graphs/rocksdb_plus.yaml + +# 5. 启动 +bin/start-hugegraph.sh +``` + +--- + +## 三、模块改造设计 + +### 3.1 保持单模块结构 + +`hugegraph-rocksdb-provider` 模块保持不变,内部简化为配置驱动: + +``` +hugegraph-rocksdb-provider/ +├── src/main/java/.../rocksdb/provider/ +│ ├── RocksDBProvider.java (接口,移除 getPriority()) +│ ├── AbstractRocksDBProvider.java (模板基类,保持不变) +│ ├── RocksDBProviderLoader.java (改为配置驱动选择) +│ ├── StandardRocksDBProvider.java (保持不变) +│ └── ToplingRocksDBProvider.java (保持不变) +├── src/main/resources/META-INF/services/ (SPI 注册,保持不变) +└── pom.xml (依赖改造) +``` + +### 3.2 POM 依赖改造 + +```xml + + + + + org.rocksdb + rocksdbjni + ${rocksdb.version} + + + + + + + + + + standard-rocksdb + + true + + + org.rocksdb + rocksdbjni + ${rocksdb.version} + + + + toplingdb + + org.apache.hugegraph + toplingdb-jni + ${toplingdb.version} + + + +``` + +`hugegraph-rocksdb-provider/pom.xml` 中依赖改为变量引用: + +```xml + + ${rocksdb.engine.groupId} + ${rocksdb.engine.artifactId} + ${rocksdb.engine.version} + +``` + +### 3.3 RocksDBProviderLoader 改造 + +从"按 priority 自动选最高"改为"按配置名精确匹配": + +```java +public class RocksDBProviderLoader { + + private final Map providerRegistry = new ConcurrentHashMap<>(); + private volatile RocksDBProvider activeProvider; + + /** + * 加载所有 SPI 注册的 Provider 到 registry + */ + public synchronized void loadProviders() { + ServiceLoader loader = ServiceLoader.load(RocksDBProvider.class); + for (RocksDBProvider provider : loader) { + providerRegistry.put(provider.getProviderName(), provider); + LOG.info("Discovered RocksDB provider: {}", provider.getProviderName()); + } + } + + /** + * 根据配置值选择并激活 Provider + */ + public synchronized RocksDBProvider selectProvider(String providerName) { + if (providerRegistry.isEmpty()) { + loadProviders(); + } + + RocksDBProvider provider = providerRegistry.get(providerName); + if (provider == null) { + throw new IllegalStateException(String.format( + "RocksDB provider '%s' not found. Available: %s. " + + "If using ToplingDB, ensure the addon is installed in lib/.", + providerName, providerRegistry.keySet())); + } + + if (!provider.isAvailable()) { + throw new IllegalStateException(String.format( + "RocksDB provider '%s' found but not available. " + + "Check native libraries and LD_PRELOAD.", + providerName)); + } + + this.activeProvider = provider; + provider.initialize(); + LOG.info("Activated RocksDB provider: {}", providerName); + return provider; + } + + /** + * 获取已激活的 Provider + */ + public RocksDBProvider getActiveProvider() { + if (activeProvider == null) { + throw new IllegalStateException( + "No RocksDB provider activated. Call selectProvider() first."); + } + return activeProvider; + } + + // static 便捷方法内部改为 getActiveProvider() + public static RocksDB openRocksDB(Options options, String dataPath, + String optionPath, Boolean openHttp) throws RocksDBException { + return getInstance().getActiveProvider() + .openRocksDB(options, dataPath, optionPath, openHttp); + } + + public static void closeRocksDB(RocksDB rocksDB) { + getInstance().getActiveProvider().closeRocksDB(rocksDB); + } +} +``` + +### 3.4 Provider 接口简化 + +```java +public interface RocksDBProvider { + + /** Provider 标识名,与 rocksdb.provider 配置值匹配 */ + String getProviderName(); // "standard" or "topling" + + /** 当前环境是否可用 */ + boolean isAvailable(); + + // open/close 方法签名保持不变... + + // 移除 getPriority() — 不再需要优先级竞争 +} +``` + +### 3.5 新增配置项 + +在 `RocksDBOptions.java`(server 和 store 两处)新增: + +```java +public static final ConfigOption PROVIDER = + new ConfigOption<>( + "rocksdb.provider", + "The RocksDB engine provider. 'standard' for vanilla RocksDB, " + + "'topling' for ToplingDB (requires addon installation).", + allowValues("standard", "topling"), + "standard" + ); +``` + +### 3.6 调用方改造 + +在 RocksDB Backend 初始化时读取配置并激活 Provider: + +```java +// RocksDBStdSessions 构造函数 或 RocksDBStoreProvider.open() 中 +String providerName = config.get(RocksDBOptions.PROVIDER); +RocksDBProviderLoader.getInstance().selectProvider(providerName); + +// 后续 open/close 调用方式不变 +RocksDBProviderLoader.openRocksDB(options, dataPath, optionPath, openHttp); +``` + +--- + +## 四、ToplingDB Addon 包 + +### 4.1 包内容 + +``` +toplingdb-addon-1.0.0-linux-x86_64.tar.gz +├── lib/ +│ └── toplingdb-jni-1.0.0.jar ← 替换 rocksdbjni-*.jar +├── native/ +│ ├── librocksdbjni-linux64.so +│ ├── libterark-zip-rocksdb-trial.so (可选) +│ └── libjemalloc.so (可选) +├── web/ +│ ├── index.html +│ └── style.css +├── conf/ +│ └── rocksdb_plus.yaml.template +└── install.sh (可选安装脚本) +``` + +### 4.2 启动脚本适配 + +`common-topling.sh` 查找逻辑改为优先使用 `native/` 目录: + +```bash +# 优先从 native/ 目录加载(addon 已安装) +if [ -d "$HUGEGRAPH_HOME/native" ] && ls "$HUGEGRAPH_HOME/native"/librocksdbjni*.so >/dev/null 2>&1; then + export LD_PRELOAD="$HUGEGRAPH_HOME/native/librocksdbjni-linux64.so" +else + # 回退:从 JAR 中提取(兼容旧方式) + jar_file=$(ls -1 "$lib_dir"/toplingdb-jni*.jar 2>/dev/null | head -1) + if [ -z "$jar_file" ]; then + jar_file=$(ls -1 "$lib_dir"/rocksdbjni*.jar 2>/dev/null | head -1) + fi + extract_so_from_jar "$jar_file" +fi +``` + +--- + +## 五、配置使用 + +### 5.1 标准 RocksDB(默认) + +```properties +backend=rocksdb +# rocksdb.provider=standard ← 默认值,可不写 +``` + +### 5.2 ToplingDB + +```properties +backend=rocksdb +rocksdb.provider=topling +rocksdb.option_path=./conf/graphs/rocksdb_plus.yaml +rocksdb.open_http=true +``` + +### 5.3 错误提示 + +| 场景 | 错误信息 | +|------|----------| +| 配置 `topling` 但未安装 addon | `RocksDB provider 'topling' not found. Available: [standard]. Install ToplingDB addon to lib/.` | +| addon JAR 在但 native lib 缺失 | `RocksDB provider 'topling' found but not available. Check native libraries and LD_PRELOAD.` | + +--- + +## 六、实施步骤 + +### Phase 1:改造 RocksDBProviderLoader(核心) + +1. 移除 `getBestProvider()` 的 priority 竞争逻辑 +2. 新增 `selectProvider(String name)` 按名精确匹配 +3. 新增 `getActiveProvider()` 替代原 `getBestProvider()` +4. `RocksDBProvider` 接口移除 `getPriority()` +5. **验证**:编译通过,配置 `standard` 时行为与原来一致 + +### Phase 2:新增配置项 + 调用方改造 + +1. `RocksDBOptions.java` (server + store) 新增 `rocksdb.provider` +2. `RocksDBStdSessions` 初始化时调用 `selectProvider(config.get(PROVIDER))` +3. `hg-store-rocksdb` 的 `RocksDBSession` 同理 +4. **验证**:`provider=standard` 正常工作 + +### Phase 3:POM 独立坐标 + Profile + +1. Root `pom.xml` 新增 `standard-rocksdb` / `toplingdb` 两个 Profile +2. `hugegraph-rocksdb-provider/pom.xml` 依赖改为 `${rocksdb.engine.*}` 变量 +3. 移除 `8.10.2-SNAPSHOT` 硬编码,标准 profile 用正式 release +4. **验证**:`mvn package` 默认无 SNAPSHOT,`mvn package -P toplingdb` 引入 ToplingDB + +### Phase 4:Addon 包 + 脚本改造 + +1. 设计 addon 包打包流程(CI) +2. 改造 `common-topling.sh` 支持 `native/` 目录 +3. 编写 `install.sh` +4. **验证**:全新部署环境通过 addon 安装 ToplingDB 正常启动 + +### Phase 5:清理 + +1. 移除 `.github/configs/settings.xml` 中 GitHub Packages 仓库(默认构建不再需要) +2. 更新配置文件模板 +3. 更新文档 +4. **验证**:完整测试矩阵 + +--- + +## 七、验证矩阵 + +| 验证项 | 操作 | 预期 | +|--------|------|------| +| 默认构建 | `mvn clean package` | 无 SNAPSHOT,无 GitHub Token,产物仅含标准 RocksDB | +| 开发者构建 | `mvn clean package -P toplingdb` | 含 ToplingDB JNI | +| 标准模式启动 | `provider=standard` | 使用原生 RocksDB | +| ToplingDB 启动 | `provider=topling` + addon 安装 | 使用 ToplingDB | +| 配置不匹配 | `provider=topling` + 未安装 addon | 明确错误信息 | +| Addon 安装 | 解压 addon + 改配置 | 无需重新编译 | +| HStore 模式 | Store 节点同验证 | 一致 | + +--- + +## 八、设计决策 + +### 为什么保持单模块? + +- `hugegraph-rocksdb-provider` 已被 `hugegraph-server/hugegraph-rocksdb` 和 `hugegraph-store/hg-store-rocksdb` 共同依赖,是两者共享 open/close 逻辑的自然位置 +- 拆成 api + standard + topling 三个模块增加了维护成本,但 Provider 实现本身代码量很小,不值得拆 +- 单模块内通过配置驱动切换,足够简洁 + +### 为什么运维走 Drop-in 而非 Maven? + +- ToplingDB 本身就需要 native lib 下载,addon 包是天然的分发单元 +- 运维不需要理解 Maven Profile,下载解压改配置即可 +- 离线环境友好 +- CI/CD 中普通构建不需要 GitHub Token + +### 为什么 `rocksdb.provider` 而非 `backend=toplingdb`? + +ToplingDB 是 RocksDB 引擎层替换,不是新的存储后端。表结构、序列化、查询全部复用 RocksDB Backend 代码。`rocksdb.provider=topling` 语义精确。 + +--- + +## 九、时间估算 + +| Phase | 工作量 | +|-------|--------| +| Phase 1:Loader 改造 | 0.5 天 | +| Phase 2:配置项 + 调用方 | 0.5 天 | +| Phase 3:POM + Profile | 0.5 天 | +| Phase 4:Addon + 脚本 | 1.5 天 | +| Phase 5:清理 | 0.5 天 | +| **合计** | **3.5 天** | diff --git a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/config/PDConfig.java b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/config/PDConfig.java index 3e68cf102c..7b677b3a15 100644 --- a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/config/PDConfig.java +++ b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/config/PDConfig.java @@ -58,6 +58,8 @@ public class PDConfig { @Value("${grpc.host}") private String host; + @Value("${rocksdb.provider:standard}") + private String provider; @Value("${rocksdb.option-path: ''}") private String optionPath; @Value("${rocksdb.open-http:false}") diff --git a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/store/HgKVStoreImpl.java b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/store/HgKVStoreImpl.java index 5d7ac2817f..a442b8327b 100644 --- a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/store/HgKVStoreImpl.java +++ b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/store/HgKVStoreImpl.java @@ -81,6 +81,8 @@ public void init(PDConfig config) { } this.optionPath = config.getOptionPath(); this.openHttp = config.getOpenHttp(); + RocksDBProviderLoader.getInstance() + .selectProviderIfNeeded(config.getProvider()); openRocksDB(dbPath); } catch (PDException e) { log.error("Failed to open data file,{}", e); diff --git a/hugegraph-rocksdb-provider/pom.xml b/hugegraph-rocksdb-provider/pom.xml index 8461b13490..d17f9c0dec 100644 --- a/hugegraph-rocksdb-provider/pom.xml +++ b/hugegraph-rocksdb-provider/pom.xml @@ -37,11 +37,11 @@ - + - org.rocksdb - rocksdbjni - 8.10.2-SNAPSHOT + ${rocksdb.engine.groupId} + ${rocksdb.engine.artifactId} + ${rocksdb.engine.version} diff --git a/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/RocksDBProvider.java b/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/RocksDBProvider.java index 8e78b4ff34..56949fa5c2 100644 --- a/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/RocksDBProvider.java +++ b/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/RocksDBProvider.java @@ -41,13 +41,6 @@ public interface RocksDBProvider { */ String getProviderName(); - /** - * Get the priority of this provider (higher priority providers are preferred) - * - * @return priority value - */ - int getPriority(); - /** * Check if this provider is available in the current environment * diff --git a/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/RocksDBProviderLoader.java b/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/RocksDBProviderLoader.java index d95bccc676..c9ff01500c 100644 --- a/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/RocksDBProviderLoader.java +++ b/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/RocksDBProviderLoader.java @@ -26,12 +26,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.ServiceLoader; import java.util.concurrent.ConcurrentHashMap; /** - * RocksDB Provider SPI Loader that manages the loading and selection - * of RocksDB providers using Java's ServiceLoader mechanism. + * RocksDB Provider Loader that manages provider discovery and selection. + * Providers are discovered via Java SPI (ServiceLoader) and selected + * explicitly by configuration name (rocksdb.provider). */ public class RocksDBProviderLoader { @@ -39,11 +44,11 @@ public class RocksDBProviderLoader { private static final RocksDBProviderLoader INSTANCE = new RocksDBProviderLoader(); - private final Map providerCache = new ConcurrentHashMap<>(); + private final Map providerRegistry = new ConcurrentHashMap<>(); + private volatile RocksDBProvider activeProvider; private volatile boolean loaded = false; private RocksDBProviderLoader() { - // Private constructor for singleton } public static RocksDBProviderLoader getInstance() { @@ -51,7 +56,7 @@ public static RocksDBProviderLoader getInstance() { } /** - * Load all available RocksDB providers using SPI + * Load all available RocksDB providers via SPI into the registry. */ public synchronized void loadProviders() { if (loaded) { @@ -63,214 +68,151 @@ public synchronized void loadProviders() { ServiceLoader serviceLoader = ServiceLoader.load(RocksDBProvider.class); for (RocksDBProvider provider : serviceLoader) { - try { - if (provider.isAvailable()) { - providerCache.put(provider.getProviderName(), provider); - LOG.info("Loaded RocksDB provider: {} (priority: {})", - provider.getProviderName(), provider.getPriority()); - } else { - LOG.warn("RocksDB provider {} is not available in current environment", - provider.getProviderName()); - } - } catch (Exception e) { - LOG.error("Failed to load RocksDB provider: {}", provider.getClass().getName(), e); - } + providerRegistry.put(provider.getProviderName(), provider); + LOG.info("Discovered RocksDB provider: {} (available: {})", + provider.getProviderName(), provider.isAvailable()); } - if (providerCache.isEmpty()) { - LOG.warn( - "No RocksDB providers found! Make sure providers are properly registered in " + - "META-INF/services"); - } else { - LOG.info("Successfully loaded {} RocksDB provider(s): {}", - providerCache.size(), providerCache.keySet()); + if (providerRegistry.isEmpty()) { + LOG.warn("No RocksDB providers found! Ensure providers are registered in " + + "META-INF/services"); } loaded = true; } /** - * Get a specific provider by name + * Select and activate a provider by name. Throws if not found or not available. * - * @param providerName provider name - * @return RocksDB provider or null if not found + * @param providerName the provider name matching rocksdb.provider config value + * @return the activated provider */ - public RocksDBProvider getProvider(String providerName) { + public synchronized RocksDBProvider selectProvider(String providerName) { if (!loaded) { loadProviders(); } - return providerCache.get(providerName); - } - - /** - * Get the best available provider based on priority - * - * @return best available RocksDB provider - */ - public RocksDBProvider getBestProvider() { - if (!loaded) { - loadProviders(); + RocksDBProvider provider = providerRegistry.get(providerName); + if (provider == null) { + throw new IllegalStateException(String.format( + "RocksDB provider '%s' not found. Available: %s. " + + "If using ToplingDB, ensure the addon is installed in lib/.", + providerName, providerRegistry.keySet())); } - if (providerCache.isEmpty()) { - throw new RuntimeException("No RocksDB providers available"); + if (!provider.isAvailable()) { + throw new IllegalStateException(String.format( + "RocksDB provider '%s' found but not available in current environment. " + + "Check native libraries and LD_PRELOAD configuration.", + providerName)); } - // Find provider with highest priority - RocksDBProvider bestProvider = null; - int highestPriority = Integer.MIN_VALUE; - - for (RocksDBProvider provider : providerCache.values()) { - if (provider.isAvailable() && provider.getPriority() > highestPriority) { - bestProvider = provider; - highestPriority = provider.getPriority(); - } - } + this.activeProvider = provider; + provider.initialize(); + LOG.info("Activated RocksDB provider: {}", providerName); + return provider; + } - if (bestProvider == null) { - throw new RuntimeException("No available RocksDB providers found"); + /** + * Idempotent version of selectProvider. If a provider is already active, skip. + * This avoids redundant initialization when multiple RocksDB instances are opened. + * + * @param providerName the provider name + */ + public synchronized void selectProviderIfNeeded(String providerName) { + if (activeProvider != null) { + return; } - - LOG.info("Auto-selected RocksDB provider: {} (priority: {})", - bestProvider.getProviderName(), bestProvider.getPriority()); - return bestProvider; + selectProvider(providerName); } /** - * Get all loaded providers + * Get the currently active provider. Throws if no provider has been selected. * - * @return collection of all providers + * @return the active provider */ - public Collection getAllProviders() { - if (!loaded) { - loadProviders(); + public RocksDBProvider getActiveProvider() { + RocksDBProvider provider = activeProvider; + if (provider == null) { + throw new IllegalStateException( + "No RocksDB provider has been activated. " + + "Ensure rocksdb.provider is configured and selectProvider() is called."); } - - return Collections.unmodifiableCollection(providerCache.values()); + return provider; } /** - * Get names of all available providers - * - * @return set of provider names + * Get names of all discovered providers. */ public Set getAvailableProviderNames() { if (!loaded) { loadProviders(); } - - return Collections.unmodifiableSet(providerCache.keySet()); + return Collections.unmodifiableSet(providerRegistry.keySet()); } /** - * Check if a specific provider is available - * - * @param providerName provider name - * @return true if provider is available + * Check if a specific provider is discovered (regardless of availability). */ public boolean isProviderAvailable(String providerName) { if (!loaded) { loadProviders(); } - - RocksDBProvider provider = providerCache.get(providerName); + RocksDBProvider provider = providerRegistry.get(providerName); return provider != null && provider.isAvailable(); } /** - * Reload all providers + * Reset loader state. After reload, selectProvider must be called again. */ public synchronized void reload() { loaded = false; - providerCache.clear(); + activeProvider = null; + providerRegistry.clear(); loadProviders(); } - // Static convenience methods + // ========== Static convenience methods ========== - /** - * Open RocksDB with simple options - * - * @param options RocksDB options - * @param dataPath database path - * @return opened RocksDB instance - * @throws RocksDBException if opening fails - */ public static RocksDB openRocksDB(Options options, String dataPath) throws RocksDBException { return openRocksDB(options, dataPath, null, null); } - /** - * Open RocksDB with options and optional parameters - * - * @param options RocksDB options - * @param dataPath database path - * @param optionPath optional path to options file - * @param openHttp optional HTTP server flag - * @return opened RocksDB instance - * @throws RocksDBException if opening fails - */ public static RocksDB openRocksDB(Options options, String dataPath, String optionPath, Boolean openHttp) throws RocksDBException { - RocksDBProvider provider = getInstance().getBestProvider(); - return provider.openRocksDB(options, dataPath, optionPath, openHttp); + return getInstance().getActiveProvider() + .openRocksDB(options, dataPath, optionPath, openHttp); } - /** - * Open RocksDB with column families - * - * @param dbOptions database options - * @param dataPath database path - * @param cfDescriptors column family descriptors - * @param cfHandles column family handles (output) - * @return opened RocksDB instance - * @throws RocksDBException if opening fails - */ public static RocksDB openRocksDB(DBOptions dbOptions, String dataPath, List cfDescriptors, List cfHandles) throws RocksDBException { return openRocksDB(dbOptions, dataPath, cfDescriptors, cfHandles, null, null); } - /** - * Open RocksDB with column families and optional parameters - * - * @param dbOptions database options - * @param dataPath database path - * @param cfDescriptors column family descriptors - * @param cfHandles column family handles (output) - * @param optionPath optional path to options file - * @param openHttp optional HTTP server flag - * @return opened RocksDB instance - * @throws RocksDBException if opening fails - */ public static RocksDB openRocksDB(DBOptions dbOptions, String dataPath, List cfDescriptors, List cfHandles, String optionPath, Boolean openHttp) throws RocksDBException { - RocksDBProvider provider = getInstance().getBestProvider(); - return provider.openRocksDB(dbOptions, dataPath, cfDescriptors, cfHandles, optionPath, - openHttp); + return getInstance().getActiveProvider() + .openRocksDB(dbOptions, dataPath, cfDescriptors, cfHandles, optionPath, openHttp); } - /** - * Close RocksDB instance - * - * @param rocksDB RocksDB instance to close - */ public static void closeRocksDB(RocksDB rocksDB) { - RocksDBProvider provider = getInstance().getBestProvider(); - provider.closeRocksDB(rocksDB); + getInstance().getActiveProvider().closeRocksDB(rocksDB); } - /** - * Get provider by name (static method) - * - * @param providerName provider name - * @return RocksDB provider or null if not found - */ public static RocksDBProvider getProviderByName(String providerName) { return getInstance().getProvider(providerName); } + + /** + * Get a provider by name without requiring it to be active. + */ + public RocksDBProvider getProvider(String providerName) { + if (!loaded) { + loadProviders(); + } + return providerRegistry.get(providerName); + } } diff --git a/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/StandardRocksDBProvider.java b/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/StandardRocksDBProvider.java index ef571c5eda..b3bd606527 100644 --- a/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/StandardRocksDBProvider.java +++ b/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/StandardRocksDBProvider.java @@ -32,18 +32,12 @@ public class StandardRocksDBProvider extends AbstractRocksDBProvider { private static final Logger LOG = LoggerFactory.getLogger(StandardRocksDBProvider.class); private static final String PROVIDER_NAME = "standard"; - private static final int PROVIDER_PRIORITY = 100; // Lower priority than ToplingDB @Override public String getProviderName() { return PROVIDER_NAME; } - @Override - public int getPriority() { - return PROVIDER_PRIORITY; - } - @Override public boolean isAvailable() { try { diff --git a/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/ToplingRocksDBProvider.java b/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/ToplingRocksDBProvider.java index 1bdb6ecc14..4f30706acc 100644 --- a/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/ToplingRocksDBProvider.java +++ b/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/ToplingRocksDBProvider.java @@ -51,7 +51,6 @@ public class ToplingRocksDBProvider extends AbstractRocksDBProvider { private static final Logger LOG = LoggerFactory.getLogger(ToplingRocksDBProvider.class); private static final String PROVIDER_NAME = "topling"; - private static final int PROVIDER_PRIORITY = 200; // Higher priority than standard private static final String SIDE_PLUGIN_REPO_CLASS = "org.rocksdb.SidePluginRepo"; // Validation constants migrated from RocksDBOptions @@ -68,11 +67,6 @@ public String getProviderName() { return PROVIDER_NAME; } - @Override - public int getPriority() { - return PROVIDER_PRIORITY; - } - @Override public boolean isAvailable() { try { diff --git a/hugegraph-rocksdb-provider/src/test/java/org/apache/hugegraph/rocksdb/provider/RocksDBProviderLoaderTest.java b/hugegraph-rocksdb-provider/src/test/java/org/apache/hugegraph/rocksdb/provider/RocksDBProviderLoaderTest.java new file mode 100644 index 0000000000..01317be678 --- /dev/null +++ b/hugegraph-rocksdb-provider/src/test/java/org/apache/hugegraph/rocksdb/provider/RocksDBProviderLoaderTest.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.rocksdb.provider; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.util.Set; + +import org.junit.Before; +import org.junit.Test; + +public class RocksDBProviderLoaderTest { + + private RocksDBProviderLoader loader; + + @Before + public void setUp() { + loader = RocksDBProviderLoader.getInstance(); + loader.reload(); + } + + /** + * Test: selectProvider("standard") should activate the StandardRocksDBProvider. + * This verifies the core config-driven selection mechanism works for the default provider. + */ + @Test + public void testSelectProviderStandard() { + RocksDBProvider provider = loader.selectProvider("standard"); + assertNotNull(provider); + assertEquals("standard", provider.getProviderName()); + assertTrue(provider.isAvailable()); + } + + /** + * Test: selectProvider("topling") should throw IllegalStateException with "not available" + * message because SidePluginRepo class is not on the test classpath. + * This ensures proper error reporting when the addon is not installed. + */ + @Test + public void testSelectProviderToplingUnavailable() { + try { + loader.selectProvider("topling"); + fail("Expected IllegalStateException for unavailable topling provider"); + } catch (IllegalStateException e) { + assertTrue(e.getMessage().contains("not available")); + assertTrue(e.getMessage().contains("topling")); + } + } + + /** + * Test: selectProvider with an unknown name should throw IllegalStateException + * with "not found" message and list available providers. + * This ensures users get actionable error info for typos or misconfiguration. + */ + @Test + public void testSelectProviderNotFound() { + try { + loader.selectProvider("nonexistent"); + fail("Expected IllegalStateException for unknown provider"); + } catch (IllegalStateException e) { + assertTrue(e.getMessage().contains("not found")); + assertTrue(e.getMessage().contains("nonexistent")); + assertTrue(e.getMessage().contains("standard")); + } + } + + /** + * Test: selectProviderIfNeeded is idempotent - calling it multiple times with the same + * or different name should not change the already-activated provider. + * This ensures that multiple RocksDB instances opening in the same JVM don't conflict. + */ + @Test + public void testSelectProviderIfNeededIdempotent() { + loader.selectProviderIfNeeded("standard"); + RocksDBProvider first = loader.getActiveProvider(); + + // Second call should be a no-op + loader.selectProviderIfNeeded("standard"); + RocksDBProvider second = loader.getActiveProvider(); + + assertSame(first, second); + } + + /** + * Test: getActiveProvider() before any selectProvider call should throw IllegalStateException. + * This ensures proper fail-fast behavior if the configuration step is missed. + */ + @Test(expected = IllegalStateException.class) + public void testGetActiveProviderBeforeSelect() { + loader.getActiveProvider(); + } + + /** + * Test: reload() clears the active provider, requiring selectProvider to be called again. + * This verifies that hot-reload scenarios work correctly. + */ + @Test + public void testReloadClearsState() { + loader.selectProvider("standard"); + assertNotNull(loader.getActiveProvider()); + + loader.reload(); + + try { + loader.getActiveProvider(); + fail("Expected IllegalStateException after reload"); + } catch (IllegalStateException e) { + assertTrue(e.getMessage().contains("No RocksDB provider has been activated")); + } + } + + /** + * Test: getAvailableProviderNames() should discover both "standard" and "topling" + * providers via SPI, regardless of their runtime availability. + */ + @Test + public void testGetAvailableProviderNames() { + Set names = loader.getAvailableProviderNames(); + assertTrue(names.contains("standard")); + assertTrue(names.contains("topling")); + } + + /** + * Test: isProviderAvailable("standard") should return true in a normal environment. + */ + @Test + public void testIsProviderAvailableStandard() { + assertTrue(loader.isProviderAvailable("standard")); + } + + /** + * Test: isProviderAvailable("topling") should return false when SidePluginRepo + * is not on the classpath (standard test environment). + */ + @Test + public void testIsProviderAvailableToplingFalse() { + assertFalse(loader.isProviderAvailable("topling")); + } + + /** + * Test: getProviderByName returns the provider instance or null without requiring activation. + * Useful for inspection/diagnostic purposes. + */ + @Test + public void testGetProviderByName() { + assertNotNull(RocksDBProviderLoader.getProviderByName("standard")); + assertNotNull(RocksDBProviderLoader.getProviderByName("topling")); + assertEquals(null, RocksDBProviderLoader.getProviderByName("unknown")); + } +} diff --git a/hugegraph-rocksdb-provider/src/test/java/org/apache/hugegraph/rocksdb/provider/StandardRocksDBProviderTest.java b/hugegraph-rocksdb-provider/src/test/java/org/apache/hugegraph/rocksdb/provider/StandardRocksDBProviderTest.java new file mode 100644 index 0000000000..0340cf9393 --- /dev/null +++ b/hugegraph-rocksdb-provider/src/test/java/org/apache/hugegraph/rocksdb/provider/StandardRocksDBProviderTest.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.rocksdb.provider; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.rocksdb.ColumnFamilyDescriptor; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.DBOptions; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; + +public class StandardRocksDBProviderTest { + + private StandardRocksDBProvider provider; + private Path tempDir; + + @Before + public void setUp() throws IOException { + provider = new StandardRocksDBProvider(); + tempDir = Files.createTempDirectory("rocksdb-test-"); + } + + @After + public void tearDown() throws IOException { + if (tempDir != null && Files.exists(tempDir)) { + Files.walk(tempDir) + .sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + } + } + + /** + * Test: getProviderName() returns "standard" - the identifier used for config matching. + */ + @Test + public void testProviderName() { + assertEquals("standard", provider.getProviderName()); + } + + /** + * Test: isAvailable() returns true when the RocksDB native library can be loaded. + * In a normal test environment with rocksdbjni on classpath, this should always pass. + */ + @Test + public void testIsAvailable() { + assertTrue(provider.isAvailable()); + } + + /** + * Test: openRocksDB with simple Options creates a working database that can be closed. + * Verifies the basic open/close lifecycle works with the standard provider. + */ + @Test + public void testOpenAndCloseRocksDB() throws RocksDBException { + Options options = new Options().setCreateIfMissing(true); + String dbPath = tempDir.resolve("test-db").toString(); + + RocksDB db = provider.openRocksDB(options, dbPath); + assertNotNull(db); + + // Verify the database is functional + db.put("key".getBytes(), "value".getBytes()); + byte[] result = db.get("key".getBytes()); + assertNotNull(result); + assertEquals("value", new String(result)); + + provider.closeRocksDB(db); + } + + /** + * Test: openRocksDB with column families creates a working database with multiple CFs. + * Verifies the DBOptions + ColumnFamilyDescriptor path works correctly. + */ + @Test + public void testOpenWithColumnFamilies() throws RocksDBException { + String dbPath = tempDir.resolve("test-cf-db").toString(); + + // First create the database with default CF + Options createOptions = new Options().setCreateIfMissing(true); + RocksDB createDb = RocksDB.open(createOptions, dbPath); + createDb.close(); + + // Now reopen with column families via the provider + DBOptions dbOptions = new DBOptions().setCreateIfMissing(true); + List cfDescriptors = new ArrayList<>(); + cfDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY)); + List cfHandles = new ArrayList<>(); + + RocksDB db = provider.openRocksDB(dbOptions, dbPath, cfDescriptors, cfHandles); + assertNotNull(db); + assertEquals(1, cfHandles.size()); + + // Write and read through the CF handle + db.put(cfHandles.get(0), "cf-key".getBytes(), "cf-value".getBytes()); + byte[] result = db.get(cfHandles.get(0), "cf-key".getBytes()); + assertEquals("cf-value", new String(result)); + + provider.closeRocksDB(db, cfHandles); + } + + /** + * Test: Passing optionPath to the standard provider does not cause an error. + * The standard provider ignores optionPath (it's a ToplingDB-only parameter) + * and should open the database normally with a warning log. + */ + @Test + public void testOpenIgnoresOptionPath() throws RocksDBException { + Options options = new Options().setCreateIfMissing(true); + String dbPath = tempDir.resolve("test-option-path").toString(); + + RocksDB db = provider.openRocksDB(options, dbPath, "/some/path.yaml", null); + assertNotNull(db); + provider.closeRocksDB(db); + } + + /** + * Test: Passing openHttp=true to the standard provider does not cause an error. + * The HTTP monitoring server is a ToplingDB-only feature, the standard provider + * should ignore it and open normally with a warning log. + */ + @Test + public void testOpenIgnoresOpenHttp() throws RocksDBException { + Options options = new Options().setCreateIfMissing(true); + String dbPath = tempDir.resolve("test-open-http").toString(); + + RocksDB db = provider.openRocksDB(options, dbPath, null, true); + assertNotNull(db); + provider.closeRocksDB(db); + } + + /** + * Test: closeRocksDB(null) is safe and does not throw any exception. + * Callers should not need to null-check before calling close. + */ + @Test + public void testCloseWithNull() { + provider.closeRocksDB(null); + } + + /** + * Test: The provider works correctly when accessed through the RocksDBProviderLoader + * static convenience methods after selectProvider("standard") is called. + * This is the actual usage path in production code. + */ + @Test + public void testOpenViaProviderLoader() throws RocksDBException { + RocksDBProviderLoader loader = RocksDBProviderLoader.getInstance(); + loader.reload(); + loader.selectProvider("standard"); + + Options options = new Options().setCreateIfMissing(true); + String dbPath = tempDir.resolve("test-loader").toString(); + + RocksDB db = RocksDBProviderLoader.openRocksDB(options, dbPath, null, null); + assertNotNull(db); + RocksDBProviderLoader.closeRocksDB(db); + } +} diff --git a/hugegraph-rocksdb-provider/src/test/java/org/apache/hugegraph/rocksdb/provider/ToplingRocksDBProviderTest.java b/hugegraph-rocksdb-provider/src/test/java/org/apache/hugegraph/rocksdb/provider/ToplingRocksDBProviderTest.java new file mode 100644 index 0000000000..64400ffd76 --- /dev/null +++ b/hugegraph-rocksdb-provider/src/test/java/org/apache/hugegraph/rocksdb/provider/ToplingRocksDBProviderTest.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.rocksdb.provider; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; + +import org.junit.Before; +import org.junit.Test; + +public class ToplingRocksDBProviderTest { + + private ToplingRocksDBProvider provider; + + @Before + public void setUp() { + provider = new ToplingRocksDBProvider(); + } + + /** + * Test: getProviderName() returns "topling" - the identifier used for config matching. + * Users set rocksdb.provider=topling to activate this provider. + */ + @Test + public void testProviderName() { + assertEquals("topling", provider.getProviderName()); + } + + /** + * Test: isAvailable() returns false when org.rocksdb.SidePluginRepo is not on the classpath. + * In a standard test environment (using vanilla rocksdbjni), ToplingDB features + * are not available. This is the expected state for most developers. + */ + @Test + public void testIsNotAvailableWithoutSidePluginRepo() { + assertFalse(provider.isAvailable()); + } + + /** + * Test: Attempting to selectProvider("topling") via the loader throws IllegalStateException + * with a message indicating the provider is found but not available. + * This simulates the error a user would see if they configure rocksdb.provider=topling + * without installing the ToplingDB addon. + */ + @Test + public void testSelectToplingThrowsWhenUnavailable() { + RocksDBProviderLoader loader = RocksDBProviderLoader.getInstance(); + loader.reload(); + + try { + loader.selectProvider("topling"); + fail("Expected IllegalStateException"); + } catch (IllegalStateException e) { + assertNotNull(e.getMessage()); + // Should mention the provider is found but not available + assertEquals(true, e.getMessage().contains("not available")); + } + } + + /** + * Test: The topling provider is still discoverable via getProviderByName even when unavailable. + * This distinguishes between "provider JAR not on classpath" (not found) + * and "provider JAR present but native libs missing" (found but not available). + */ + @Test + public void testProviderDiscoverableViaLoader() { + RocksDBProviderLoader loader = RocksDBProviderLoader.getInstance(); + loader.reload(); + + RocksDBProvider found = loader.getProvider("topling"); + assertNotNull(found); + assertEquals("topling", found.getProviderName()); + assertFalse(found.isAvailable()); + } +} diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java index 6f54e2f439..4c3973ff31 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java @@ -86,6 +86,15 @@ public static synchronized RocksDBOptions instance() { "rocksdb-data/wal" ); + public static final ConfigOption PROVIDER = + new ConfigOption<>( + "rocksdb.provider", + "The RocksDB engine provider. 'standard' for vanilla RocksDB, " + + "'topling' for ToplingDB (requires addon installation).", + allowValues("standard", "topling"), + "standard" + ); + public static final ConfigOption OPTION_PATH = new ConfigOption<>( "rocksdb.option_path", diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java index b1307af572..3086c41f18 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java @@ -375,6 +375,10 @@ private void ingestExternalFile() throws RocksDBException { private static OpenedRocksDB openRocksDB(HugeConfig config, String dataPath, String walPath) throws RocksDBException { + // Select provider based on explicit configuration + RocksDBProviderLoader.getInstance() + .selectProviderIfNeeded(config.get(RocksDBOptions.PROVIDER)); + // Init options Options options = new Options(); RocksDBStdSessions.initOptions(config, options, options, options, options); @@ -398,6 +402,10 @@ private static OpenedRocksDB openRocksDB(HugeConfig config, List cfNames, String dataPath, String walPath) throws RocksDBException { + // Select provider based on explicit configuration + RocksDBProviderLoader.getInstance() + .selectProviderIfNeeded(config.get(RocksDBOptions.PROVIDER)); + // Old CFs should always be opened Set mergedCFs = RocksDBStdSessions.mergeOldCFs(dataPath, cfNames); diff --git a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBOptions.java b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBOptions.java index 77b635557d..0abade5294 100644 --- a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBOptions.java +++ b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBOptions.java @@ -56,6 +56,14 @@ public class RocksDBOptions extends OptionHolder { disallowEmpty(), false ); + public static final ConfigOption PROVIDER = + new ConfigOption<>( + "rocksdb.provider", + "The RocksDB engine provider. 'standard' for vanilla RocksDB, " + + "'topling' for ToplingDB (requires addon installation).", + allowValues("standard", "topling"), + "standard" + ); public static final ConfigOption OPTION_PATH = new ConfigOption<>( "rocksdb.option_path", diff --git a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java index ca40bf1271..c7499a0d34 100644 --- a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java +++ b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java @@ -413,6 +413,9 @@ public boolean tableIsExist(String table) { } private void openRocksDB(String dbDataPath, long version) { + // Select provider based on explicit configuration + RocksDBProviderLoader.getInstance() + .selectProviderIfNeeded(hugeConfig.get(RocksDBOptions.PROVIDER)); if (dbDataPath.endsWith(File.separator)) { this.dbPath = dbDataPath + this.graphName; diff --git a/pom.xml b/pom.xml index dc99d26640..8f41db2b3c 100644 --- a/pom.xml +++ b/pom.xml @@ -96,6 +96,11 @@ UTF-8 bash 1.5.0 + 8.10.2 + 1.0.0 + org.rocksdb + rocksdbjni + ${rocksdb.version} @@ -410,5 +415,14 @@ osx-x86_64 + + + toplingdb + + org.apache.hugegraph + toplingdb-jni + ${toplingdb.version} + + From bb8bef12585d5636a7ba09fa708b72a40826704f Mon Sep 17 00:00:00 2001 From: ken <2979602290@qq.com> Date: Thu, 9 Jul 2026 11:31:23 +0800 Subject: [PATCH 49/51] fix(rocksdb): add provider mismatch detection and move selection to constructor initialization configuration --- .../provider/RocksDBProviderLoader.java | 10 +++--- .../provider/RocksDBProviderLoaderTest.java | 33 +++++++++++++++---- .../provider/ToplingRocksDBProviderTest.java | 2 +- .../store/rocksdb/RocksDBStdSessions.java | 12 +++---- 4 files changed, 37 insertions(+), 20 deletions(-) diff --git a/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/RocksDBProviderLoader.java b/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/RocksDBProviderLoader.java index c9ff01500c..f272c22ebf 100644 --- a/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/RocksDBProviderLoader.java +++ b/hugegraph-rocksdb-provider/src/main/java/org/apache/hugegraph/rocksdb/provider/RocksDBProviderLoader.java @@ -121,6 +121,12 @@ public synchronized RocksDBProvider selectProvider(String providerName) { */ public synchronized void selectProviderIfNeeded(String providerName) { if (activeProvider != null) { + if (!activeProvider.getProviderName().equals(providerName)) { + throw new IllegalStateException(String.format( + "RocksDB provider '%s' is already active, cannot switch to '%s'. " + + "Ensure all components use the same rocksdb.provider configuration.", + activeProvider.getProviderName(), providerName)); + } return; } selectProvider(providerName); @@ -202,10 +208,6 @@ public static void closeRocksDB(RocksDB rocksDB) { getInstance().getActiveProvider().closeRocksDB(rocksDB); } - public static RocksDBProvider getProviderByName(String providerName) { - return getInstance().getProvider(providerName); - } - /** * Get a provider by name without requiring it to be active. */ diff --git a/hugegraph-rocksdb-provider/src/test/java/org/apache/hugegraph/rocksdb/provider/RocksDBProviderLoaderTest.java b/hugegraph-rocksdb-provider/src/test/java/org/apache/hugegraph/rocksdb/provider/RocksDBProviderLoaderTest.java index 01317be678..baad040892 100644 --- a/hugegraph-rocksdb-provider/src/test/java/org/apache/hugegraph/rocksdb/provider/RocksDBProviderLoaderTest.java +++ b/hugegraph-rocksdb-provider/src/test/java/org/apache/hugegraph/rocksdb/provider/RocksDBProviderLoaderTest.java @@ -20,6 +20,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -86,7 +87,7 @@ public void testSelectProviderNotFound() { /** * Test: selectProviderIfNeeded is idempotent - calling it multiple times with the same - * or different name should not change the already-activated provider. + * name should not change the already-activated provider. * This ensures that multiple RocksDB instances opening in the same JVM don't conflict. */ @Test @@ -94,13 +95,31 @@ public void testSelectProviderIfNeededIdempotent() { loader.selectProviderIfNeeded("standard"); RocksDBProvider first = loader.getActiveProvider(); - // Second call should be a no-op + // Second call with same name should be a no-op loader.selectProviderIfNeeded("standard"); RocksDBProvider second = loader.getActiveProvider(); assertSame(first, second); } + /** + * Test: selectProviderIfNeeded with a different name than the already-active provider + * should throw IllegalStateException to surface configuration inconsistency. + */ + @Test + public void testSelectProviderIfNeededMismatchThrows() { + loader.selectProviderIfNeeded("standard"); + + try { + loader.selectProviderIfNeeded("topling"); + fail("Expected IllegalStateException for provider mismatch"); + } catch (IllegalStateException e) { + assertTrue(e.getMessage().contains("already active")); + assertTrue(e.getMessage().contains("standard")); + assertTrue(e.getMessage().contains("topling")); + } + } + /** * Test: getActiveProvider() before any selectProvider call should throw IllegalStateException. * This ensures proper fail-fast behavior if the configuration step is missed. @@ -158,13 +177,13 @@ public void testIsProviderAvailableToplingFalse() { } /** - * Test: getProviderByName returns the provider instance or null without requiring activation. + * Test: getProvider returns the provider instance or null without requiring activation. * Useful for inspection/diagnostic purposes. */ @Test - public void testGetProviderByName() { - assertNotNull(RocksDBProviderLoader.getProviderByName("standard")); - assertNotNull(RocksDBProviderLoader.getProviderByName("topling")); - assertEquals(null, RocksDBProviderLoader.getProviderByName("unknown")); + public void testGetProvider() { + assertNotNull(loader.getProvider("standard")); + assertNotNull(loader.getProvider("topling")); + assertNull(loader.getProvider("unknown")); } } diff --git a/hugegraph-rocksdb-provider/src/test/java/org/apache/hugegraph/rocksdb/provider/ToplingRocksDBProviderTest.java b/hugegraph-rocksdb-provider/src/test/java/org/apache/hugegraph/rocksdb/provider/ToplingRocksDBProviderTest.java index 64400ffd76..76bd5e285b 100644 --- a/hugegraph-rocksdb-provider/src/test/java/org/apache/hugegraph/rocksdb/provider/ToplingRocksDBProviderTest.java +++ b/hugegraph-rocksdb-provider/src/test/java/org/apache/hugegraph/rocksdb/provider/ToplingRocksDBProviderTest.java @@ -75,7 +75,7 @@ public void testSelectToplingThrowsWhenUnavailable() { } /** - * Test: The topling provider is still discoverable via getProviderByName even when unavailable. + * Test: The topling provider is still discoverable via getProvider even when unavailable. * This distinguishes between "provider JAR not on classpath" (not found) * and "provider JAR present but native libs missing" (found but not available). */ diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java index 3086c41f18..7ce7dff2b2 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java @@ -92,6 +92,8 @@ public RocksDBStdSessions(HugeConfig config, String database, String store, this.dataPath = dataPath; this.walPath = walPath; + RocksDBProviderLoader.getInstance() + .selectProviderIfNeeded(config.get(RocksDBOptions.PROVIDER)); this.rocksdb = RocksDBStdSessions.openRocksDB(config, dataPath, walPath); this.refCount = new AtomicInteger(1); } @@ -105,6 +107,8 @@ public RocksDBStdSessions(HugeConfig config, String database, String store, this.dataPath = dataPath; this.walPath = walPath; + RocksDBProviderLoader.getInstance() + .selectProviderIfNeeded(config.get(RocksDBOptions.PROVIDER)); this.rocksdb = RocksDBStdSessions.openRocksDB(config, cfNames, dataPath, walPath); this.refCount = new AtomicInteger(1); @@ -375,10 +379,6 @@ private void ingestExternalFile() throws RocksDBException { private static OpenedRocksDB openRocksDB(HugeConfig config, String dataPath, String walPath) throws RocksDBException { - // Select provider based on explicit configuration - RocksDBProviderLoader.getInstance() - .selectProviderIfNeeded(config.get(RocksDBOptions.PROVIDER)); - // Init options Options options = new Options(); RocksDBStdSessions.initOptions(config, options, options, options, options); @@ -402,10 +402,6 @@ private static OpenedRocksDB openRocksDB(HugeConfig config, List cfNames, String dataPath, String walPath) throws RocksDBException { - // Select provider based on explicit configuration - RocksDBProviderLoader.getInstance() - .selectProviderIfNeeded(config.get(RocksDBOptions.PROVIDER)); - // Old CFs should always be opened Set mergedCFs = RocksDBStdSessions.mergeOldCFs(dataPath, cfNames); From 486b399ea201f1b73691252ec7a8fd3270aae941 Mon Sep 17 00:00:00 2001 From: WaterWhisperer Date: Thu, 9 Jul 2026 14:36:49 +0800 Subject: [PATCH 50/51] fix(toplingdb): align explicit provider configuration --- .specs/hugegraph-server/ToplingDB/design.md | 85 +++++++++++-------- .specs/hugegraph-server/ToplingDB/task.md | 16 ++-- docs/toplingdb/toplingdb-troubleshooting.md | 64 ++++++++++++-- docs/toplingdb/toplingdb.md | 81 ++++++++++++++---- .../apache/hugegraph/pd/config/PDConfig.java | 2 +- .../src/assembly/static/conf/application.yml | 1 + .../static/conf/graphs/hugegraph.properties | 1 + .../assembly/static/conf/application-pd.yml | 2 + 8 files changed, 187 insertions(+), 65 deletions(-) diff --git a/.specs/hugegraph-server/ToplingDB/design.md b/.specs/hugegraph-server/ToplingDB/design.md index 4a4cc378b6..83000164d5 100644 --- a/.specs/hugegraph-server/ToplingDB/design.md +++ b/.specs/hugegraph-server/ToplingDB/design.md @@ -40,34 +40,39 @@ sequenceDiagram ### RocksDB Startup Logic -Use reflection to detect whether ToplingDB APIs are available. If present, attempt to start the storage engine using ToplingDB; otherwise, fall back to standard RocksDB APIs. +HugeGraph routes the main RocksDB open/close paths through `RocksDBProviderLoader`. +Providers are discovered via Java SPI, but the active provider is selected explicitly by +`rocksdb.provider`. `standard` uses vanilla RocksDB; `topling` requires +`org.rocksdb.SidePluginRepo` to be available at runtime. ```mermaid sequenceDiagram autonumber - participant Store as HugeGraph Store + participant Store as HugeGraph Server/PD/Store participant Config as HugeConfig - participant Sessions as RocksDBStdSessions - participant SPR as SidePluginRepo (Reflection) + participant Loader as RocksDBProviderLoader + participant Provider as Configured Provider participant Repo as Repo Instance participant Rocks as RocksDB - Store->>Config: Read OPTION_PATH / OPEN_HTTP - Store->>Sessions: Create Sessions(..., optionPath, openHttp) - alt optionPath provided and SPR available - Sessions->>SPR: Reflectively load/create Repo - Sessions->>Repo: importAutoFile(optionPath) - Sessions->>Repo: open (with JSON descriptor) - opt openHttp is true and instance is GRAPH_STORE - Sessions->>Repo: startHttpServer() + Store->>Config: Read PROVIDER / OPTION_PATH / OPEN_HTTP + Store->>Loader: selectProviderIfNeeded(provider) + Store->>Loader: openRocksDB(..., optionPath, openHttp) + Loader->>Provider: openRocksDB(...) + alt provider is standard + Provider->>Rocks: RocksDB.open(...) + else provider is topling and optionPath valid + Provider->>Repo: Reflectively create SidePluginRepo + Provider->>Repo: importAutoFile(optionPath) + Provider->>Repo: openDB(JSON descriptor) + opt Server openHttp is true and instance is GRAPH_STORE + Provider->>Repo: startHttpServer() end - Sessions->>Rocks: Get RocksDB instance (with CF handles) - Sessions-->>Store: Return OpenedRocksDB(repo, handles) - else optionPath not provided or SPR unavailable - Sessions->>Rocks: Standard open() - Sessions-->>Store: Return OpenedRocksDB(null, handles) + else provider is topling but optionPath pre-validation fails + Provider->>Rocks: RocksDB.open(...) end - note over Store,Sessions: On shutdown, if repo exists, call repo.closeAllDB() + Loader-->>Store: Return RocksDB instance + note over Store,Loader: On shutdown, close through the active provider; Topling closes Repo when present. ``` ## Involved Modules @@ -168,19 +173,30 @@ Both `init-hugegraph.sh` and `start-hugegraph.sh` now invoke `preload-topling.sh ### HugeGraph Configuration Options for RocksDB -Two new configuration options were added to `hugegraph.properties`: `option_path` for the YAML file and `open_http` to enable the Web Server. +Three RocksDB provider options were added: `provider` selects standard RocksDB or ToplingDB, +`option_path` points to the YAML file, and `open_http` enables the Web Server. ```properties # rocksdb backend config #rocksdb.data_path=/path/to/disk #rocksdb.wal_path=/path/to/disk -#rocksdb.option_path=./conf/graphs/rocksdb_plus.yaml +#rocksdb.provider=topling +#rocksdb.option_path=./conf/graphs/rocksdb_server.yaml #rocksdb.open_http=true ``` Java-side parsing and default values: ```java +public static final ConfigOption PROVIDER = + new ConfigOption<>( + "rocksdb.provider", + "The RocksDB engine provider. 'standard' for vanilla RocksDB, " + + "'topling' for ToplingDB (requires addon installation).", + allowValues("standard", "topling"), + "standard" + ); + public static final ConfigOption OPTION_PATH = new ConfigOption<>( "rocksdb.option_path", @@ -200,29 +216,30 @@ public static final ConfigOption OPEN_HTTP = ### ToplingDB Startup Logic -When `option_path` is configured and the JAR contains ToplingDB APIs, HugeGraph will load the YAML file and start ToplingDB. Otherwise, it falls back to standard RocksDB. +When `rocksdb.provider=topling`, `option_path` is configured, pre-validation passes, and the JAR +contains ToplingDB APIs, HugeGraph will load the YAML file and start ToplingDB. If +`rocksdb.provider=standard`, HugeGraph uses standard RocksDB. -To keep port configuration simple, the Web Server is only enabled for the `GRAPH_STORE` instance. +For HugeGraph Server, the Web Server is only enabled for the `GRAPH_STORE` instance. PD and +Store pass their component-level HTTP flag to the provider. ```mermaid flowchart LR - A[Start Initialization] --> B{Is optionPath provided?} - - B -- No --> Z[Use standard RocksDB to open DB] --> O[Finish standard init] - - B -- Yes --> C{Is SidePluginRepo class available?} - C -- No --> Z + A[Start Initialization] --> P{rocksdb.provider} + P -- standard --> Z[Use StandardRocksDBProvider] --> O[Finish standard init] + P -- topling --> S{Is SidePluginRepo available?} + S -- No --> F[Fail startup] + S -- Yes --> B{Is optionPath provided and valid?} - C -- Yes --> D[Set useTopling = true] --> E["Load SidePluginRepo class and call importAutoFile(optionPath)"] + B -- No --> Z + B -- Yes --> E["ToplingRocksDBProvider calls importAutoFile(optionPath)"] - E --> K{Is openHttp true?} + E --> K{Should this component start HTTP?} K -- No --> M[Finish Topling init] - K -- Yes --> L{Is dbName GRAPH_STORE?} - L -- No --> M - L -- Yes --> N["startHttpServer()"] --> M + K -- Yes --> N["startHttpServer()"] --> M ``` ## Design Decisions and Rationale -1. **Why is the Web Server only started for GRAPH_STORE?** +1. **Why does HugeGraph Server only start the Web Server for GRAPH_STORE?** - All graph data is stored in GRAPH_STORE, and performance tuning and observability are primarily focused on this instance. diff --git a/.specs/hugegraph-server/ToplingDB/task.md b/.specs/hugegraph-server/ToplingDB/task.md index 7ae83eff1b..1fcd615630 100644 --- a/.specs/hugegraph-server/ToplingDB/task.md +++ b/.specs/hugegraph-server/ToplingDB/task.md @@ -12,7 +12,7 @@ When using an IDE such as IntelliJ IDEA, you need to configure the following env ```shell LD_LIBRARY_PATH=/path/to/your/library:$LD_LIBRARY_PATH -LD_PRELOAD=libjemalloc.so:librocksdbjni-linux64.so +LD_PRELOAD=/path/to/libjemalloc.so:/path/to/your/library/librocksdbjni-linux64.so ``` When running from the terminal, simply use `init-store.sh` and `start-hugegraph.sh`, as `preload-topling.sh` is already embedded in these scripts. @@ -25,9 +25,10 @@ When running from the terminal, simply use `init-store.sh` and `start-hugegraph. ## 2. Compatibility with ToplingDB and Standard RocksDB -- [x] **2.1 Modify openRocksDB logic in RocksDBStdSession** - - Use reflection to detect whether the current JAR contains ToplingDB APIs; if so, start the storage engine using ToplingDB - - If not available, fall back to the standard RocksDB API for engine startup +- [x] **2.1 Add a RocksDB provider SPI and route main open/close paths through it** + - Use `RocksDBProviderLoader` to discover standard RocksDB and ToplingDB providers + - Select the active provider explicitly with `rocksdb.provider` + - Use reflection in `ToplingRocksDBProvider` to detect whether the current JAR contains `org.rocksdb.SidePluginRepo` ## 3. Add Configuration Options for ToplingDB in HugeGraph @@ -40,7 +41,12 @@ When running from the terminal, simply use `init-store.sh` and `start-hugegraph. - Type: boolean, used to specify whether to enable the ToplingDB Web Server - Allow users to configure Web Server activation via `rocksdb.open_http` in `hugegraph.properties` - The Web Server port is defined in the YAML file specified by `option_path`, under `http.listening_ports` - - For simplicity, the Web Server is only enabled for the `GRAPH_STORE` instance that stores graph data + - For HugeGraph Server, the Web Server is only enabled for the `GRAPH_STORE` instance that stores graph data + +- [x] **3.3 Add `rocksdb.provider` configuration** + - Type: string, valid values: `standard` and `topling` + - Default: `standard` + - Prevent ToplingDB from being enabled implicitly just because a ToplingDB-capable JAR is on the classpath ## 4. End-to-End Performance Testing diff --git a/docs/toplingdb/toplingdb-troubleshooting.md b/docs/toplingdb/toplingdb-troubleshooting.md index 68d8072c46..21d9a4dd4e 100644 --- a/docs/toplingdb/toplingdb-troubleshooting.md +++ b/docs/toplingdb/toplingdb-troubleshooting.md @@ -2,13 +2,13 @@ ## Issues -### Issue 1: Startup Failure Due to YAML Format Error +### Issue 1: YAML Configuration Error Sample log output: ```java -2025-10-15 01:55:50 [db-open-1] [INFO] o.a.h.b.s.r.RocksDBStdSessions - SidePluginRepo found. Will attempt to open multi CFs RocksDB using Topling plugin. -21:1: (891B):ERROR: +2025-10-15 01:55:50 [db-open-1] [INFO] o.a.h.r.p.ToplingRocksDBProvider - SidePluginRepo found. Will attempt to open default CF RocksDB using Topling. +21:1: (891B):ERROR: sideplugin/rockside/3rdparty/rapidyaml/src/c4/yml/parse.cpp:3310: ERROR parsing yml: parse error: incorrect indentation? ``` @@ -18,19 +18,49 @@ sideplugin/rockside/3rdparty/rapidyaml/src/c4/yml/parse.cpp:3310: ERROR parsing 2. Validate YAML syntax: ```bash - python -c "import yaml; yaml.safe_load(open('conf/graphs/rocksdb_plus.yaml'))" + python -c "import yaml; yaml.safe_load(open('conf/graphs/rocksdb_server.yaml'))" ``` 3. Review the specific error message in the logs for further clues. +HugeGraph performs basic path and YAML validation before enabling ToplingDB. If this +pre-validation fails, HugeGraph falls back to standard RocksDB behavior inside the ToplingDB +provider. If the file passes basic YAML validation but `SidePluginRepo.importAutoFile()` or +`openDB()` fails, startup fails with the corresponding ToplingDB error and the YAML file should +be fixed. + --- -### Issue 2: Web Server Port Conflict +### Issue 2: ToplingDB Provider Not Available + +When `rocksdb.provider=topling` is configured but the runtime `rocksdbjni` JAR does not contain +`org.rocksdb.SidePluginRepo`, HugeGraph fails provider activation instead of silently switching +to the standard provider. + +**Solution**: + +1. If you want standard RocksDB, set: + + ```properties + rocksdb.provider=standard + ``` + +2. If you want ToplingDB, confirm that Maven or the deployment addon provides a ToplingDB-capable + `rocksdbjni` artifact. +3. Inspect the runtime JAR: + + ```bash + jar tf lib/rocksdbjni-*.jar | grep org/rocksdb/SidePluginRepo.class + ``` + +--- + +### Issue 3: Web Server Port Conflict Sample log output: ```java -2025-10-15 01:57:34 [db-open-1] [INFO] o.a.h.b.s.r.RocksDBStdSessions - SidePluginRepo found. Will attempt to open multi CFs RocksDB using Topling plugin. +2025-10-15 01:57:34 [db-open-1] [INFO] o.a.h.r.p.ToplingRocksDBProvider - SidePluginRepo found. Will attempt to open default CF RocksDB using Topling. 2025-10-15 01:57:34 [db-open-1] [ERROR] o.a.h.b.s.r.RocksDBStore - Failed to open RocksDB 'rocksdb-data/data/g' org.rocksdb.RocksDBException: rocksdb::Status rocksdb::SidePluginRepo::StartHttpServer(): null context when constructing CivetServer. Possible problem binding to port. at org.rocksdb.SidePluginRepo.startHttpServer(Native Method) ~[rocksdbjni-8.10.2-20250804.074027-4.jar:?] @@ -49,7 +79,7 @@ org.rocksdb.RocksDBException: rocksdb::Status rocksdb::SidePluginRepo::StartHttp --- -### Issue 3: Database Initialization Failure +### Issue 4: Database Initialization Failure This error indicates the database lock file cannot be acquired, possibly due to insufficient write permissions or another process holding the lock: @@ -64,7 +94,8 @@ Caused by: org.rocksdb.RocksDBException: While lock file: rocksdb-data/data/m/LO 1. Confirm the configuration file path is correct: ```properties - rocksdb.option_path=./conf/graphs/rocksdb_plus.yaml + rocksdb.provider=topling + rocksdb.option_path=./conf/graphs/rocksdb_server.yaml ``` 2. Check permissions on the data directory to ensure the running user has read/write access. @@ -76,6 +107,23 @@ Caused by: org.rocksdb.RocksDBException: While lock file: rocksdb-data/data/m/LO --- +### Issue 5: Ubuntu 24.04+ libaio Compatibility + +Some ToplingDB native libraries may expect the legacy `libaio.so.1` name, while Ubuntu 24.04+ +ships the time64 library as `libaio.so.1t64`. The preload script currently creates a +compatibility symlink when running on Ubuntu 24.04+ and `libaio.so.1` is missing. + +If startup fails with a missing `libaio.so.1` error, install the system package first: + +```bash +sudo apt-get install libaio1t64 +``` + +Then retry startup. If your environment does not allow scripts to use `sudo`, create the +compatibility symlink manually or ask the system administrator to provide it. + +--- + ## Log Analysis ### Enable Debug Logging diff --git a/docs/toplingdb/toplingdb.md b/docs/toplingdb/toplingdb.md index 2a0566fde0..dc0635a080 100644 --- a/docs/toplingdb/toplingdb.md +++ b/docs/toplingdb/toplingdb.md @@ -17,7 +17,7 @@ ToplingDB extends RocksDB with several advanced features: ## Motivation -Introduce a new optional storage component in HugeGraph to support [ToplingDB](https://github.com/topling/toplingdb)), a configurable and observable extension of the RocksDB storage engine. +Introduce a new optional storage component in HugeGraph to support [ToplingDB](https://github.com/topling/toplingdb), a configurable and observable extension of the RocksDB storage engine. ToplingDB resolves key limitations in HugeGraph’s current `rocksdbjni` integration, which relies heavily on hard-coding parameters and lacks runtime configurability and observability. @@ -39,9 +39,25 @@ Enable users to select ToplingDB via configuration, allowing tuning parameters t ## Design +### Provider Selection + +HugeGraph uses the `hugegraph-rocksdb-provider` module to keep RocksDB and ToplingDB startup +logic behind a shared provider interface. Providers are discovered through Java SPI, but the +active provider is selected explicitly by configuration: + +```properties +rocksdb.provider=standard +``` + +Use `standard` for vanilla RocksDB and `topling` for ToplingDB. ToplingDB also requires a valid +`rocksdb.option_path` and a runtime `rocksdbjni` JAR that contains +`org.rocksdb.SidePluginRepo`. + ### Configuration Parameters -To support ToplingDB in HugeGraph, two new configuration parameters have been introduced: `rocksdb.option_path` and `rocksdb.open_http`. These options allow users to configure RocksDB parameters and enable real-time observability. +To support ToplingDB in HugeGraph, three configuration parameters have been introduced: +`rocksdb.provider`, `rocksdb.option_path`, and `rocksdb.open_http`. These options allow users to +select the provider, configure RocksDB parameters, and enable real-time observability. #### `rocksdb.option_path`: External YAML Configuration @@ -52,18 +68,37 @@ This parameter allows users to specify a YAML file that defines ToplingDB settin - **Usage**: Add the following line to your `hugegraph.properties` file: ```properties - rocksdb.option_path=./conf/graphs/rocksdb_plus.yaml + rocksdb.provider=topling + rocksdb.option_path=./conf/graphs/rocksdb_server.yaml ``` The specified YAML file will be automatically loaded during database initialization if ToplingDB is available. - For security reasons, HugeGraph only allows YAML files to be stored under the `$HUGEGRAPH_HOME/conf/graphs` directory. + PD and Store use their own YAML configuration files: + + ```yaml + rocksdb: + provider: topling + option-path: ./conf/graphs/rocksdb_pd.yaml + open-http: true + ``` + + ```yaml + rocksdb: + provider: topling + option_path: ./conf/rocksdb_store.yaml + open_http: false + ``` + + For security reasons, HugeGraph only allows YAML files to be stored under the `$HUGEGRAPH_HOME/conf/` directory. For details on the YAML structure and supported configuration fields, please refer to [SidePlugin](https://github.com/topling/sideplugin-wiki-en/wiki). - **Implementation**: During initialization, HugeGraph checks whether the configured JAR contains ToplingDB APIs. If so, it uses reflection to load the SidePluginRepo class and calls `importAutoFile(optionPath)` to parse the YAML file. The resulting configuration is applied to the RocksDB instance. -- **Fallback**: If the YAML file is not provided or ToplingDB is unavailable, HugeGraph will fall back to standard RocksDB behavior. +- **Fallback**: If `rocksdb.provider=standard`, HugeGraph uses standard RocksDB. If + `rocksdb.provider=topling` is configured but ToplingDB is unavailable, startup fails with a + provider availability error instead of silently switching providers. #### `rocksdb.open_http`: Enable Web Server for Observability @@ -89,9 +124,14 @@ This boolean flag controls whether the embedded Web Server in ToplingDB should b When adjusting this setting, users should carefully manage port and network access permissions to avoid potential security incidents. To preview the Web Server interface and its layout, see [Web Server](https://github.com/topling/sideplugin-wiki-en/wiki/WebView). -- **Implementation**: If `open_http` is set to true and the database instance is `GRAPH_STORE`, HugeGraph invokes `startHttpServer()` on the ToplingDB repo object. This exposes a browser-accessible dashboard for monitoring RocksDB internals. +- **Implementation**: For HugeGraph Server's RocksDB backend, `open_http=true` is forwarded to + the provider only when the opened database instance is `GRAPH_STORE`. The provider then invokes + `startHttpServer()` on the ToplingDB repo object. This exposes a browser-accessible dashboard + for monitoring RocksDB internals. -- **Scope**: For simplicity, the Web Server is only enabled for the `GRAPH_STORE` instance, which holds the main graph data. +- **Scope**: In HugeGraph Server, the Web Server is only enabled for the `GRAPH_STORE` instance, + which holds the main graph data. PD and Store pass their component-level `open-http` / + `open_http` setting to the provider. - **Security**: The Web Server does **not** provide built-in authentication. In production environments, configure firewalls or network access controls carefully to prevent unauthorized access. @@ -99,14 +139,21 @@ This boolean flag controls whether the embedded Web Server in ToplingDB should b To support ToplingDB without introducing hard dependencies, HugeGraph uses Java reflection to detect and load enhanced APIs at runtime. -During initialization, HugeGraph checks whether the current JAR contains the class `com.topling.sideplugin.SidePluginRepo`. If present, it assumes ToplingDB is available and proceeds to: - -1. **Load the SidePluginRepo class via reflection** This avoids compile-time coupling and allows fallback to standard RocksDB if the class is missing. - - If the ToplingDB API cannot be found, HugeGraph silently falls back to the standard RocksDB API for startup. -2. **Invoke** `importAutoFile(optionPath)` This method parses the YAML configuration file specified by `rocksdb.option_path` to configure storage engine parameters. - - If the `option_path` is incorrect or parsing fails, ToplingDB throws an error and terminates the startup process. -3. **Call** `open()` **with a JSON descriptor** The parsed configuration is converted to a JSON structure and passed to the ToplingDB engine to initialize the database. -4. **Optionally start the Web Server** If `rocksdb.open_http` is true and the instance is `GRAPH_STORE`, HugeGraph invokes `startHttpServer()` via reflection to enable observability. +During initialization, HugeGraph selects the configured provider. When `rocksdb.provider=topling`, +it checks whether the current JAR contains the class `org.rocksdb.SidePluginRepo`. If present, it +uses ToplingDB as follows: + +1. **Load the SidePluginRepo class via reflection**. This avoids compile-time coupling. + - If the ToplingDB API cannot be found while the ToplingDB provider is selected, startup fails. +2. **Validate** `option_path` and the YAML file before enabling ToplingDB. + - If the path is missing, unsafe, unreadable, outside `conf/`, or fails basic YAML parsing, + HugeGraph falls back to standard RocksDB behavior inside the ToplingDB provider. +3. **Invoke** `importAutoFile(optionPath)`. This method lets `SidePluginRepo` parse the YAML + configuration file specified by `rocksdb.option_path` and configure storage engine parameters. + - If ToplingDB initialization fails after this point, startup fails with the corresponding + RocksDB or SidePlugin error. +4. **Call** `open()` **with a JSON descriptor**. The parsed configuration is converted to a JSON structure and passed to the ToplingDB engine to initialize the database. +5. **Optionally start the Web Server**. In HugeGraph Server, if `rocksdb.open_http` is true and the instance is `GRAPH_STORE`, HugeGraph invokes `startHttpServer()` via reflection to enable observability. PD and Store pass their component-level HTTP flag to the provider. - If the Web Server cannot be started due to misconfiguration **or if the specified HTTP port is already in use**, ToplingDB throws an error and the startup process is terminated This design ensures that ToplingDB can be integrated as an optional enhancement, without breaking compatibility or requiring changes to the core HugeGraph codebase. @@ -116,7 +163,7 @@ This design ensures that ToplingDB can be integrated as an optional enhancement, ### For Users The way users operate remains unchanged by default, and adding ToplingDB configuration provides additional functionality. -The ToplingDB integration is fully embedded into the existing startup scripts (`init-store.sh` and `start-hugegraph.sh`). Users only need to set `rocksdb.option_path` to specify the YAML file path and adjust its contents as needed to tune the storage engine. +The ToplingDB integration is fully embedded into the existing startup scripts (`init-store.sh` and `start-hugegraph.sh`). Users only need to set `rocksdb.provider=topling` and `rocksdb.option_path` to specify the YAML file path, then adjust its contents as needed to tune the storage engine. ### For Developers @@ -167,7 +214,7 @@ Developers need to make two adjustments to enable ToplingDB during development: ```bash LD_LIBRARY_PATH="/path/to/your/library:${LD_LIBRARY_PATH}" - LD_PRELOAD="libjemalloc.so:librocksdbjni.so" + LD_PRELOAD="/path/to/libjemalloc.so:/path/to/your/library/librocksdbjni-linux64.so" ``` These steps ensure that ToplingDB loads correctly in development environments and behaves consistently with production deployments. diff --git a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/config/PDConfig.java b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/config/PDConfig.java index 7b677b3a15..d7a75caef1 100644 --- a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/config/PDConfig.java +++ b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/config/PDConfig.java @@ -60,7 +60,7 @@ public class PDConfig { @Value("${rocksdb.provider:standard}") private String provider; - @Value("${rocksdb.option-path: ''}") + @Value("${rocksdb.option-path:}") private String optionPath; @Value("${rocksdb.open-http:false}") private Boolean openHttp; diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/application.yml b/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/application.yml index 8828674874..53f8074fb8 100644 --- a/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/application.yml +++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/application.yml @@ -40,6 +40,7 @@ grpc: # The service address of grpc needs to be changed to the actual local IPv4 address when deploying. host: 127.0.0.1 # rocksdb: + # provider: topling # option-path: ./conf/graphs/rocksdb_pd.yaml # open-http: true diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/hugegraph.properties b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/hugegraph.properties index 25ad5ddf59..0498cb7664 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/hugegraph.properties +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/hugegraph.properties @@ -43,6 +43,7 @@ search.text_analyzer_mode=INDEX # rocksdb backend config #rocksdb.data_path=/path/to/disk #rocksdb.wal_path=/path/to/disk +#rocksdb.provider=topling #rocksdb.option_path=./conf/graphs/rocksdb_server.yaml #rocksdb.open_http=true diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/conf/application-pd.yml b/hugegraph-store/hg-store-dist/src/assembly/static/conf/application-pd.yml index 737617e0d9..f11c34844b 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/conf/application-pd.yml +++ b/hugegraph-store/hg-store-dist/src/assembly/static/conf/application-pd.yml @@ -32,6 +32,8 @@ rocksdb: write_buffer_size: 32000000 # For each rocksdb, the number of memtables reaches this value for writing to disk. min_write_buffer_number_to_merge: 16 + # The RocksDB engine provider, optional values: standard, topling + # provider: topling # Configuration file path for rocksdb/toplingdb # option_path: ./conf/rocksdb_store.yaml # Whether to start Topling's HTTP service From fe8c6b0ed0da1deb2a04c384e766e9d9d2ca5a43 Mon Sep 17 00:00:00 2001 From: WaterWhisperer Date: Thu, 9 Jul 2026 14:47:43 +0800 Subject: [PATCH 51/51] fix(ci): align ToplingDB CI setup --- .github/workflows/codeql-analysis.yml | 1 + docs/toplingdb/toplingdb.md | 7 ++- .../apache/hugegraph/pd/config/PDConfig.java | 6 +- .../src/assembly/static/conf/application.yml | 2 +- .../src/assembly/static/bin/common-topling.sh | 58 +++++++++++++------ .../src/assembly/travis/install-rocksdb.sh | 29 ++++++++-- .../scripts/dependency/known-dependencies.txt | 4 +- 7 files changed, 76 insertions(+), 31 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 52f3d2a93e..e45d6a7c8a 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -20,6 +20,7 @@ jobs: permissions: actions: read contents: read + packages: read security-events: write strategy: diff --git a/docs/toplingdb/toplingdb.md b/docs/toplingdb/toplingdb.md index dc0635a080..3a30c0f51d 100644 --- a/docs/toplingdb/toplingdb.md +++ b/docs/toplingdb/toplingdb.md @@ -37,6 +37,11 @@ By supporting ToplingDB, HugeGraph empowers users with greater control over stor Enable users to select ToplingDB via configuration, allowing tuning parameters through YAML files without recompilation and real-time monitoring via Web Server—without sacrificing compatibility with existing RocksDB APIs. +## Platform Scope + +The current ToplingDB native integration supports Linux x86_64 only. +On macOS, Linux arm64/aarch64, and other non-x86_64 platforms, HugeGraph should continue to use the standard RocksDB `rocksdbjni` dependency instead of preloading ToplingDB native libraries. + ## Design ### Provider Selection @@ -79,7 +84,7 @@ This parameter allows users to specify a YAML file that defines ToplingDB settin ```yaml rocksdb: provider: topling - option-path: ./conf/graphs/rocksdb_pd.yaml + option-path: ./conf/rocksdb_pd.yaml open-http: true ``` diff --git a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/config/PDConfig.java b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/config/PDConfig.java index d7a75caef1..730038fb6c 100644 --- a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/config/PDConfig.java +++ b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/config/PDConfig.java @@ -59,11 +59,11 @@ public class PDConfig { private String host; @Value("${rocksdb.provider:standard}") - private String provider; + private String provider = "standard"; @Value("${rocksdb.option-path:}") - private String optionPath; + private String optionPath = ""; @Value("${rocksdb.open-http:false}") - private Boolean openHttp; + private Boolean openHttp = false; @Value("${license.verify-path}") private String verifyPath; diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/application.yml b/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/application.yml index 53f8074fb8..32f14d9277 100644 --- a/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/application.yml +++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/application.yml @@ -41,7 +41,7 @@ grpc: host: 127.0.0.1 # rocksdb: # provider: topling - # option-path: ./conf/graphs/rocksdb_pd.yaml + # option-path: ./conf/rocksdb_pd.yaml # open-http: true server: diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh index 43004ba6d9..9703576eb6 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh @@ -153,27 +153,28 @@ function ensure_libaio_symlink() { function download_and_verify() { local url=$1 local filepath=$2 - local expected_md5=$3 + local expected_sha256=$3 + local actual_sha256 - if [[ -f $filepath ]]; then - echo "File $filepath exists. Verifying MD5 checksum..." - actual_md5=$(md5sum $filepath | awk '{ print $1 }') - if [[ $actual_md5 != $expected_md5 ]]; then - echo "MD5 checksum verification failed for $filepath. Expected: $expected_md5, but got: $actual_md5" + if [[ -f "$filepath" ]]; then + echo "File $filepath exists. Verifying SHA-256 checksum..." + actual_sha256=$(sha256sum "$filepath" | awk '{ print $1 }') + if [[ "$actual_sha256" != "$expected_sha256" ]]; then + echo "SHA-256 checksum verification failed for $filepath. Expected: $expected_sha256, but got: $actual_sha256" echo "Deleting $filepath..." - rm -f $filepath + rm -f "$filepath" else - echo "MD5 checksum verification succeeded for $filepath." + echo "SHA-256 checksum verification succeeded for $filepath." return 0 fi fi echo "Downloading $filepath..." - curl -L -o $filepath $url + curl -fL -o "$filepath" "$url" - actual_md5=$(md5sum $filepath | awk '{ print $1 }') - if [[ $actual_md5 != $expected_md5 ]]; then - echo "MD5 checksum verification failed for $filepath after download. Expected: $expected_md5, but got: $actual_md5" + actual_sha256=$(sha256sum "$filepath" | awk '{ print $1 }') + if [[ "$actual_sha256" != "$expected_sha256" ]]; then + echo "SHA-256 checksum verification failed for $filepath after download. Expected: $expected_sha256, but got: $actual_sha256" return 1 fi @@ -181,7 +182,7 @@ function download_and_verify() { } function download_and_setup_jemalloc() { - local arch lib_file download_url expected_md5 system_lib top + local arch lib_file download_url expected_sha256 system_lib top top=$1 # Prefer system-installed jemalloc if available @@ -224,19 +225,19 @@ function download_and_setup_jemalloc() { if [[ $arch == "aarch64" || $arch == "arm64" ]]; then lib_file="$top/bin/libjemalloc_aarch64.so" download_url="${GITHUB}/apache/hugegraph-doc/raw/binary-1.5/dist/server/libjemalloc_aarch64.so" - expected_md5="2a631d2f81837f9d5864586761c5e380" + expected_sha256="6b7e6099b6da798829c6ce6fcb55a787508841edd52446332a73300889dcd1dc" elif [[ $arch == "x86_64" ]]; then lib_file="$top/bin/libjemalloc.so" download_url="${GITHUB}/apache/hugegraph-doc/raw/binary-1.5/dist/server/libjemalloc.so" - expected_md5="fd61765eec3bfea961b646c269f298df" + expected_sha256="53b25e8626e1605cbd8b60befb3431cabc1b8851a54285e0dda412796feab67d" else echo "Unsupported architecture: $arch" return 1 fi # Download and verify jemalloc library (fallback when system lib not found) - if download_and_verify "$download_url" "$lib_file" "$expected_md5"; then - if [[ ":${LD_PRELOAD:-}:" != *"libjemalloc.so:"* ]]; then + if download_and_verify "$download_url" "$lib_file" "$expected_sha256"; then + if [[ ":${LD_PRELOAD:-}:" != *":${lib_file}:"* ]]; then export LD_PRELOAD="${lib_file}${LD_PRELOAD:+:$LD_PRELOAD}" fi else @@ -248,6 +249,25 @@ function download_and_setup_jemalloc() { function preload_toplingdb() { local lib_dir="$1" local dest_dir="$2" + local os_name machine_arch + + # NOTE: The current ToplingDB rocksdbjni snapshot bundles Linux x86_64 native libraries. + # Linux arm64/aarch64 support requires a matching native rocksdbjni artifact. + os_name="$(uname -s)" + if [ "$os_name" != "Linux" ]; then + echo "[common-topling] Skip ToplingDB native preload on non-Linux platform: $os_name" >&2 + return 0 + fi + machine_arch="$(uname -m)" + case "$machine_arch" in + x86_64 | amd64) + ;; + *) + echo "[common-topling] Skip ToplingDB native preload on unsupported platform: $os_name/$machine_arch" >&2 + return 0 + ;; + esac + local top="$(cd "$lib_dir"/../ && pwd)" local jar_file @@ -275,7 +295,9 @@ function preload_toplingdb() { { echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" echo "LD_PRELOAD=$LD_PRELOAD" - echo "SERVER_VERSION_DIR=$SERVER_VERSION_DIR" + if [ -n "${SERVER_VERSION_DIR:-}" ]; then + echo "SERVER_VERSION_DIR=$SERVER_VERSION_DIR" + fi } >> "$GITHUB_ENV" || true echo "[common-topling] Exported LD_LIBRARY_PATH and LD_PRELOAD to GITHUB_ENV" >&2 || true fi diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh index 05f1085ffe..8531ca4182 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh @@ -16,15 +16,23 @@ # limitations under the License. # -set -Eeuo pipefail +if [ "$(uname -s)" != "Linux" ]; then + echo "[install-rocksdb] Skip ToplingDB native preload on non-Linux platform: $(uname -s)" + return 0 2>/dev/null || exit 0 +fi + +ORIG_SHELL_FLAGS="$-" +ORIG_PIPEFAIL="$(set -o | awk '$1 == "pipefail" { print $2 }')" +ORIG_ERR_TRAP="$(trap -p ERR || true)" # Save original IFS to avoid leaking into parent shell when sourced ORIG_IFS="${IFS}" +set -Eeuo pipefail IFS=$'\n\t' # Unified error capture for easy positioning trap 'echo "[install-rocksdb] error at line ${LINENO}: ${BASH_COMMAND}" >&2' ERR VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) -SERVER_VERSION_DIR="$(pwd)/hugegraph-server/apache-hugegraph-server-incubating-$VERSION" +SERVER_VERSION_DIR="$(pwd)/hugegraph-server/apache-hugegraph-server-$VERSION" SERVER_BIN="$SERVER_VERSION_DIR/bin" SERVER_LIB="$SERVER_VERSION_DIR/lib" INSTALL_DEST_DIR="$SERVER_VERSION_DIR/library" @@ -46,8 +54,19 @@ source "$SERVER_BIN/common-topling.sh" type preload_toplingdb >/dev/null 2>&1 || { echo "Error: function preload_toplingdb not found" >&2; exit 1; } preload_toplingdb "$SERVER_LIB" "$INSTALL_DEST_DIR" -# Reset shell options to prevent affecting the parent shell when sourced -set +Eeuo pipefail -trap - ERR # Restore original IFS IFS="$ORIG_IFS" +if [ -n "$ORIG_ERR_TRAP" ]; then + eval "$ORIG_ERR_TRAP" +else + trap - ERR +fi +# Reset shell options to prevent affecting the parent shell when sourced +case "$ORIG_SHELL_FLAGS" in *e*) set -e ;; *) set +e ;; esac +case "$ORIG_SHELL_FLAGS" in *u*) set -u ;; *) set +u ;; esac +case "$ORIG_SHELL_FLAGS" in *E*) set -E ;; *) set +E ;; esac +if [ "$ORIG_PIPEFAIL" = "on" ]; then + set -o pipefail +else + set +o pipefail +fi diff --git a/install-dist/scripts/dependency/known-dependencies.txt b/install-dist/scripts/dependency/known-dependencies.txt index d515630b0d..c0242d56b0 100644 --- a/install-dist/scripts/dependency/known-dependencies.txt +++ b/install-dist/scripts/dependency/known-dependencies.txt @@ -318,8 +318,6 @@ jnr-ffi-2.1.7.jar jnr-x86asm-1.0.2.jar joda-time-2.10.8.jar joni-2.2.1.jar -jraft-core-1.3.11.jar -jraft-core-1.3.13.jar jraft-core-1.3.14.jar jraft-core-1.3.9.jar json-path-2.5.0.jar @@ -512,7 +510,7 @@ psjava-0.1.19.jar reporter-config-base-3.0.3.jar reporter-config3-3.0.3.jar rewriting-9.0-9.0.20190305.jar -rocksdbjni-8.10.2-SNAPSHOT.jar +rocksdbjni-8.10.2.jar scala-java8-compat_2.12-0.8.0.jar scala-library-2.12.7.jar scala-reflect-2.12.7.jar