Merge DevBranch: SymDB2

read All the symbols to memory, provide Super Fast dlsym
switch to SymDB2, which is faster than before

Author: KawaiiNahida

.github/workflows/build.yml

@@ -45,17 +45,17 @@ jobs:
         unzip D:/cc.zip
       shell: bash
 
-    - name: Download RoDB.exe
+    - name: Copy SymDB2 Gen
+      working-directory: ${{env.GITHUB_WORKSPACE}}
       run: |
-        curl -L -o D:/out/release/RoDB.exe https://github.com/Sysca11/BDXCore/raw/master/RoDB.exe
+        cp -a Tool/SymDB2.exe D:/out/release
       shell: bash
 
     - name: Copy Output Files
       working-directory: ${{env.GITHUB_WORKSPACE}}
       run: |
         cp -a x64/Release/*.dll D:/out/release
-        cp -a D:/cc/x64_release/ChakraCore.dll D:/out/release
-        
+        cp -a D:/cc/x64_release/ChakraCore.dll D:/out/release   
       shell: bash
 
     - name: Copy DevOutput Files

LiteLoader/LiteLoader.cpp

@@ -117,7 +117,6 @@ void FixUpCWD() {
 void startWBThread();
 bool versionCommand(CommandOrigin const &ori, CommandOutput &outp);
 void updateCheck();
-
 static void entry(bool fixcwd) {
     //禁止弹窗，便于自动重启
     SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX | SEM_NOALIGNMENTFAULTEXCEPT);

LiteLoader/dlsym.cpp

@@ -15,136 +15,237 @@
 using std::list;
 using std::string, std::string_view;
 using std::unordered_map, std::vector;
-
-typedef unsigned long long hash_t;
-hash_t BKDR(const char *_ch, size_t sz) {
-    unsigned char *ch = (unsigned char *)_ch;
-    hash_t rv         = 0;
-    for (int i = 0; i < sz; ++i) {
-        rv = rv * 127 - 36 + ch[i];
+struct aphash {
+    size_t operator()(const string &x) const {
+        uint64_t rval = 0;
+        for (size_t i = 0; x[i]; ++i) {
+            if (i & 1) {
+                rval ^= (~((rval << 11) ^ x[i] ^ (rval >> 5)));
+            } else {
+                rval ^= (~((rval << 7) ^ x[i] ^ (rval >> 3)));
+            }
+        }
+        return rval;
     }
-    return rv;
-}
-struct hashval {
-    string key;
-    hash_t hash;
-    string data;
-    hashval() { hash = 0; }
-    hashval(string &&a, hash_t b, string &&c)
-        : key(std::forward<string>(a)), hash(b), data(std::forward<string>(c)) {}
 };
-struct RoDB_R {
-    std::ifstream fp;
-    unsigned int cnt_bucket;
-    unsigned int bucket[4096];  // offset
-    unsigned int data_off;
-    RoDB_R(const char *path) {
-        fp.open(path, std::ios::binary);
-#define ppch(x) ((char *)&x)
-        fp.read(ppch(cnt_bucket), 4);
-        fp.read(ppch(data_off), 4);
-        fp.read(ppch(bucket[0]), 4 * (size_t)cnt_bucket);
+struct SymDBBase {
+    // constexpr static int SEGMENT_MAX = 8;
+    constexpr static int SEGMENT_HASH_ORDERED = 0;
+    constexpr static int SEGMENT_RVA_INT      = 1;
+    constexpr static int SEGMENT_STRINGS_IDX  = 2;
+    constexpr static int SEGMENT_STRINGS      = 3;
+    constexpr static int SEGMENT_COUNT        = 4;
+};
+#include <vector>
+typedef int s32;
+typedef int64_t s64;
+typedef uint64_t u64;
+typedef uint32_t u32;
+typedef uint16_t u16;
+typedef uint8_t u8;
+template <int BKDR_MUL = 131, int BKDR_ADD = 0>
+constexpr u64 BKDRHash(const char *x, int len) {
+    u64 rval = 0;
+    for (size_t i = 0; i < len; ++i) {
+        rval *= BKDR_MUL;
+        rval += x[i];
+        rval += BKDR_ADD;
     }
-    void read(void *buf, unsigned int off, size_t sz) {
-        fp.seekg(std::streampos(off));
-        fp.read((char *)buf, sz);
+    return rval;
+}
+#include <algorithm>
+#include <fstream>
+
+struct SymDBReader : SymDBBase {
+    int SEGOFF[SEGMENT_COUNT], SIZSEG[SEGMENT_COUNT];
+    std::ifstream ifs;
+    u64 *hashes;
+    u64 *hashes_end;
+    template <typename T>
+    T _pfread(int off = -1) {
+        if (off != -1)
+            ifs.seekg({off});
+        T rv;
+        ifs.read((char *)&rv, sizeof(T));
+        return rv;
     }
-    void getstr(string &buf, unsigned int off) {
-        if (off != 0) {
-            fp.seekg(std::streampos(off));
+    SymDBReader(const char *fn) {
+        ifs.open(fn, std::ios::binary);
+        for (int &off : SEGOFF) {
+            off = _pfread<int>();
+        }
+        for (int &siz : SIZSEG) {
+            siz = _pfread<int>();
+        }
+        if (SEGOFF[0] != 8 * SEGMENT_COUNT) {
+            printf("SymDB format error detected!!\n");
         }
-        char ch;
-        while ((ch = fp.get()) != 0)
-            buf.append(1, ch);
+        int seg_begin = SEGOFF[SEGMENT_HASH_ORDERED];
+        int seg_siz   = SIZSEG[SEGMENT_HASH_ORDERED];
+        char *buf     = (char *)malloc(seg_siz);
+        ifs.seekg({seg_begin});
+        ifs.read(buf, seg_siz);
+        hashes     = (u64 *)buf;
+        hashes_end = hashes + (seg_siz / 8);
     }
-    string val2key(unsigned int rva) {
-        fp.seekg(std::streampos(data_off));
-        string name;
-        name.reserve(8192);
-        while (!fp.eof()) {
-            int ch = fp.get();
-            if (ch == 0) {
-                unsigned int dst;
-                fp.read((char *)&dst, 4);
-                if (dst == rva) {
-                    return name;
-                } else {
-                    name.clear();
+    ~SymDBReader() { free(hashes); }
+    int lookupInternal(u64 hash, int length, u64 hash2, const char *symname) {
+        // step 1:lookup internal index
+        u64 *lbound = std::lower_bound(hashes, hashes_end, hash);
+        if (lbound == hashes_end || *lbound != hash)
+            return -1;
+        int internalIdx = int(std::distance(hashes, lbound));
+        int matchCount  = 0;
+        for (u64 *start = lbound; start < hashes_end; ++start) {
+            if (*start == hash)
+                ++matchCount;
+            else
+                break;
+        }
+        // printf("internalIdx [%d,%d)\n", internalIdx, internalIdx+matchCount);
+        // step 2:load string offsets
+        ifs.seekg({SEGOFF[SEGMENT_STRINGS_IDX] + internalIdx * 4});
+        int string_off[256];
+        for (int i = 0; i <= matchCount; ++i) {
+            // string_off[i] = _pfread<int>(SEGOFF[SEGMENT_STRINGS_IDX] + (internalIdx + i) * 4);
+            string_off[i] = _pfread<int>();
+        }
+        int idx_hits[128];
+        int siz_idx_hits = 0;
+        for (int i = 0; i < matchCount; ++i) {
+            int strlength = string_off[i + 1] - string_off[i];
+            if (length == strlength) {
+                idx_hits[siz_idx_hits++] = internalIdx + i;
+            }
+        }
+        if (siz_idx_hits == 0) {
+            return -1;
+        }
+        int retval = -1;
+        for (int i = 0; i < siz_idx_hits; ++i) {
+            int nowidx = idx_hits[i];
+            int strptr = string_off[i] + SEGOFF[SEGMENT_STRINGS];
+            ifs.seekg({strptr});
+            if (symname != nullptr) {
+                static constexpr auto CMPSTR = [](std::ifstream &ifs, const char *symname) -> bool {
+                    for (char const *compare = symname; *compare; ++compare) {
+                        if (*compare != ifs.get())
+                            return false;
+                    }
+                    return true;
+                };
+                if (CMPSTR(ifs, symname)) {
+                    return _pfread<int>(SEGOFF[SEGMENT_RVA_INT] + nowidx * 4);
                 }
             } else {
-                name.push_back(ch);
+                static constexpr auto HASHSTR = [](std::ifstream &ifs, int len) -> u64 {
+                    u64 rv = 0;
+                    while (len-- > 0) {
+                        rv = 131 * rv + char(ifs.get());
+                    }
+                    return rv;
+                };
+                if (HASHSTR(ifs, length) == hash2) {
+                    if (retval != -1) {
+                        printf("hash coll detected!\n");
+                        exit(1);
+                    }
+                    retval = _pfread<int>(SEGOFF[SEGMENT_RVA_INT] + nowidx * 4);
+                }
             }
         }
-        return "(nil)";
+        return retval;
     }
-    bool _cmp(string_view key) {
-        for (uint32_t i = 0; i < key.size(); ++i) {
-            int ch = fp.get();
-            if (ch != key[i]) {
-                return false;
-            }
-        }
-        return fp.get() == 0;
+    int getsym(const char *name) {
+        int len = int(strlen(name));
+        return lookupInternal(do_hash(name, len), len, 0, name);
     }
-    unsigned int get(string_view key) {  // return file offset
-        auto hash  = BKDR(key.data(), key.size());
-        auto bkoff = bucket[hash % cnt_bucket];
-        fp.seekg(std::streampos(bkoff));
-        vector<unsigned int> tolookup;
-        tolookup.reserve(512);
-        while (1) {
-            hash_t hs;
-            unsigned int off;
-            fp.read(ppch(hs), 8);
-            fp.read(ppch(off), 4);
-            if (off == 0xffffffff)
-                break;
-            if (hs == hash)
-                tolookup.push_back(off + data_off);
-            /*
-            if (hs == hash) {
-                    off += data_off;
-                    auto pos = fp.tellg();
-                    string key_now;
-                    getstr(key_now, off);
-                    if (key == key_now) {
-                            return off + (unsigned int)key.size() + 1;
-                    }
-                    fp.seekg(pos);
-            }*/
+    std::string rva2name(int rva) {
+        int siz_rva = SIZSEG[SEGMENT_RVA_INT];
+        char *buf   = new char[siz_rva];
+        ifs.seekg({SEGOFF[SEGMENT_RVA_INT]});
+        ifs.read(buf, siz_rva);
+        int *need  = (int *)buf;
+        int *end   = need + siz_rva / 4;
+        int *found = std::find(need, end, rva);
+        if (found == end) {
+            return "(nil)";
         }
-        for (auto off : tolookup) {
-            fp.seekg({off});
-            if (_cmp(key)) {
-                return off + (uint32_t)key.size() + 1;
-            }
+        int internalId = int(std::distance(need, found));
+        int soff       = _pfread<int>(SEGOFF[SEGMENT_STRINGS_IDX] + internalId * 4);
+        int slen       = _pfread<int>(SEGOFF[SEGMENT_STRINGS_IDX] + internalId * 4 + 4) - soff;
+        ifs.seekg({int(SEGOFF[SEGMENT_STRINGS] + soff)});
+        std::string rv;
+        rv.resize(slen, 0);
+        ifs.read(rv.data(), slen);
+        delete[] buf;
+        return rv;
+    }
+    void dumpall(unordered_map<string, int, aphash>* hashMap) {
+        int siz_rva = SIZSEG[SEGMENT_RVA_INT];
+        char *buf   = new char[siz_rva];
+        ifs.seekg({SEGOFF[SEGMENT_RVA_INT]});
+        ifs.read(buf, siz_rva);
+        int *need = (int *)buf;
+        int *end  = need + siz_rva / 4;
+        for (int a = 0; need + a != end; a++) {
+            int *rva       = need + a;
+            int internalId = int(std::distance(need, rva));
+            int soff       = _pfread<int>(SEGOFF[SEGMENT_STRINGS_IDX] + internalId * 4);
+            int slen       = _pfread<int>(SEGOFF[SEGMENT_STRINGS_IDX] + internalId * 4 + 4) - soff;
+            ifs.seekg({int(SEGOFF[SEGMENT_STRINGS] + soff)});
+            char *rv = new char[slen];
+            ifs.read((char *)rv, slen);
+            hashMap->insert({rv, *rva});
+            //hashMap[rv] = *rva;
+            // printf("[%08d] %s\n", *rva, rv.c_str());
+            delete[] rv;
         }
-        return 0;
+
+        delete[] buf;
     }
 };
-static RoDB_R *pdb;
+int fnstat = 0;
+static SymDBReader *SymDB;
 static uintptr_t BaseAdr;
-string ptr2name(void *ptr) {
-    unsigned int va = uint32_t(((uintptr_t)ptr - BaseAdr));
-    return pdb->val2key(va);
+unordered_map<string, int, aphash> *FuncMap;
+void InitFastDlsym() {
+    printf("[Info] Loading Symbols\n");
+    unordered_map<string, int, aphash> *realFuncMap = new unordered_map<string, int, aphash>;
+    SymDB->dumpall(realFuncMap);
+    fnstat = 1;
+    SymDB    = nullptr;
+    SymDB    = new SymDBReader("bedrock_server.symdb2");
+    FuncMap                        = realFuncMap;
+    printf("[Info] FastDlsymInited <%zd>\n", realFuncMap->size());
 }
 void *dlsym_real(const char *x) {
-    if (pdb == nullptr) {
-        if (!std::filesystem::exists("bedrock_server.symdb")) {
-            printf("SymDB not found\ntry to run RoDB.exe\n");
+    if (SymDB == nullptr) {
+        if (!std::filesystem::exists("bedrock_server.symdb2")) {
+            printf("SymDB not found\ntry to run SymDB2.exe\n");
             std::this_thread::sleep_for(std::chrono::seconds(10));
             exit(1);
         }
-        pdb     = new RoDB_R("bedrock_server.symdb");
+        SymDB     = new SymDBReader("bedrock_server.symdb2");
         BaseAdr = (uintptr_t)GetModuleHandle(NULL);
         static_assert(sizeof(GetModuleHandle(NULL)) == 8);
     }
-    auto rv = pdb->get(x);
-    if (!rv)
-        return nullptr;
-    unsigned int rva;
-    pdb->read(&rva, rv, 4);
-    return (void *)(BaseAdr + rva);
+    if (fnstat == 0)
+        InitFastDlsym();
+    for (;fnstat == 0;) {
+        std::this_thread::sleep_for(std::chrono::seconds(1));
+    } 
+        if (fnstat == 1) {
+            auto iter = FuncMap->find(x);
+            if (iter != FuncMap->end()) {
+                //std::cout<<x<<" : "<<iter->second<<" == "<<rv<<std::endl;
+                return (void *)(BaseAdr + iter->second);
+            }
+        }
+        auto rv = SymDB->getsym(x);
+        if (!rv)
+            return nullptr;
+        return (void *)(BaseAdr + rv);
 }
 inline static void HookFunction__begin() {
     DetourTransactionBegin();

LiteLoader/pch.h

@@ -17,6 +17,7 @@
 #include <api\xuidreg\xuidreg.h>
 #include <lbpch.h>
 #include <filesystem>
+#include <map>
 #include "framework.h"
 // CheckUpdate.cpp
 #include <rapidjson\document.h>

LiteLoader/xuidreg/xuidreg.cpp

@@ -79,7 +79,7 @@ void initAll() {
         auto _xuid = offPlayer::getXUIDString(sp);
         xuid_t xuid;
         if (_xuid.size() <= 1)
-            xuid = do_hash(name);
+            xuid = do_hash(name.c_str());
         else
             xuid = std::stoull(_xuid);
         insert(xuid, name);