feat: auto download tiktoken files

Author: RobinQu

modules/instinct-core/include/tools/HashUtils.hpp

@@ -40,7 +40,7 @@ namespace INSTINCT_CORE_NS {
         requires IsHashImplementation<Hash>
         static std::string HashForStream(std::istream& input_stream) {
             Hash hash;
-            static constexpr size_t BUFFER_SIZE = 144*7*1024;
+            static constexpr size_t BUFFER_SIZE = 1024;
             char buf[BUFFER_SIZE];
             while (input_stream) {
                 input_stream.read(buf, BUFFER_SIZE);

modules/instinct-core/include/tools/file_vault/FileSystemFileVault.hpp

@@ -137,6 +137,7 @@ namespace INSTINCT_CORE_NS {
         std::filesystem::path root_directory_;
         std::unordered_map<std::string, FileVaultResourceProviderPtr> resources_;
         std::mutex write_mutex_;
+        std::unordered_map<std::string, std::mutex> resource_mutexes_;
 
     public:
         explicit FileSystemFileVault(std::filesystem::path root_directory)
@@ -159,6 +160,9 @@ namespace INSTINCT_CORE_NS {
             return std::async(std::launch::async, [&] {
                 assert_true(resources_.contains(named_resource),
                             fmt::format("Resource {} should exist in vault", named_resource));
+
+                // lock and get resource provider
+                std::lock_guard lock(resource_mutexes_[named_resource]);
                 const auto resource_provider = resources_.at(named_resource);
 
                 // get resource paths

modules/instinct-examples/doc-agent/src/doc-agent.cpp

@@ -141,7 +141,7 @@ namespace insintct::exmaples::doc_agent {
             LOG_INFO("CreateChunkedMultiVectorRetriever");
 
             // default to use tiktoken tokenizer
-            auto tokenizer = TiktokenTokenizer::MakeGPT4Tokenizer("/Users/robinqu/Downloads/cl100k_base.tiktoken");
+            auto tokenizer = TiktokenTokenizer::MakeGPT4Tokenizer();
 
             const auto child_spliter = CreateRecursiveCharacterTextSplitter(tokenizer, {.chunk_size = retriever_options.child_chunk_size});
             if (retriever_options.parent_chunk_size > 0) {

modules/instinct-llm/include/tokenizer/TiktokenTokenizer.hpp

@@ -11,6 +11,7 @@
 
 #include "GPT2BPEFileReader.hpp"
 #include "TiktokenBPEFileReader.hpp"
+#include "tools/file_vault/FileSystemFileVault.hpp"
 
 
 namespace INSTINCT_LLM_NS {
@@ -69,6 +70,29 @@ namespace INSTINCT_LLM_NS {
             return std::make_shared<TiktokenTokenizer>(bpe_ranks, vocab, UnicodeString::fromUTF8(config.pat_str), config.special_tokens, byte_shuffle);
         }
 
+        static TokenizerPtr MakeGPT2Tokenizer() {
+            static bool ONCE = false;
+            if(!ONCE) {
+                FetchHttpGetResourceToFileVault(
+                    DEFAULT_FILE_VAULT,
+                    "tiktoken/gpt2_vocab.bpe",
+                    "https://openaipublic.blob.core.windows.net/gpt-2/encodings/main/vocab.bpe",
+                    {.algorithm = kSHA256, .expected_value = "1ce1664773c50f3e0cc8842619a93edc4624525b728b188a9e0be33b7726adc5"}
+                ).wait();
+
+                FetchHttpGetResourceToFileVault(
+                    DEFAULT_FILE_VAULT,
+                    "tiktoken/gpt2_encoder.json",
+                    "https://openaipublic.blob.core.windows.net/gpt-2/encodings/main/encoder.json",
+                    {.algorithm = kSHA256, .expected_value = "196139668be63f3b5d6574427317ae82f612a97c5d1cdaf36ed2256dbf636783"}
+                ).wait();
+            }
+
+            const auto entry1 = DEFAULT_FILE_VAULT->GetResource("tiktoken/gpt2_vocab.bpe").get();
+            const auto entry2 = DEFAULT_FILE_VAULT->GetResource("tiktoken/gpt2_vocab.bpe").get();
+            return MakeGPT2Tokenizer(entry1.local_path, entry2.local_path);
+        }
+
         static TokenizerPtr MakeGPT2Tokenizer(
             const std::filesystem::path& bpe_file_path,
             const std::filesystem::path& encoder_json_file_path) {
@@ -84,6 +108,20 @@ namespace INSTINCT_LLM_NS {
             });
         }
 
+        static TokenizerPtr MakeGPT4Tokenizer() {
+            static bool ONCE = false;
+            if(!ONCE) {
+                FetchHttpGetResourceToFileVault(
+                    DEFAULT_FILE_VAULT,
+                    "tiktoken/cl100k_base.tiktoken",
+                    "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken",
+                    {.algorithm = kSHA256, .expected_value = "223921b76ee99bde995b7ff738513eef100fb51d18c93597a113bcffe865b2a7"}
+                ).wait();
+            }
+            const auto entry = DEFAULT_FILE_VAULT->GetResource("tiktoken/cl100k_base.tiktoken").get();
+            return MakeGPT4Tokenizer(entry.local_path);
+        }
+
         static TokenizerPtr MakeGPT4Tokenizer(
             const std::filesystem::path& tiktoken_bpe_file_path
             ) {