diff --git a/.github/helper-bot/.clang-format b/.github/helper-bot/.clang-format new file mode 100644 index 00000000..97f01a4e --- /dev/null +++ b/.github/helper-bot/.clang-format @@ -0,0 +1,4 @@ +ColumnLimit: 120 +SortIncludes: false +BreakStringLiterals: false +AllowShortFunctionsOnASingleLine: Empty \ No newline at end of file diff --git a/.github/helper-bot/.gitignore b/.github/helper-bot/.gitignore new file mode 100644 index 00000000..c21a078f --- /dev/null +++ b/.github/helper-bot/.gitignore @@ -0,0 +1,11 @@ +# server +bds-* +# built PDB and server disassembly analysis executables +*.exe +# intermediates from pdba/disa +rodata.* +strings.* +stage*.txt +# update1 +collected*.json +updatedBody.md \ No newline at end of file diff --git a/.github/helper-bot/README.txt b/.github/helper-bot/README.txt new file mode 100644 index 00000000..e6edd670 --- /dev/null +++ b/.github/helper-bot/README.txt @@ -0,0 +1,8 @@ +1. index.js -- ran initially on CRON to check for updates +2. update1.js -- runs bedrock-protocol client against the updated server to collect data from server->client. Also runs a behavior pack to extract block data. +3. disa.exe -- disassembly analysis for Minecraft bedrock edition server binary (combining data from both Linux/Win binaries) + * x86 disassembly for the server software with symbol information is analogus to decompiling the Minecraft Java Edition + and running various extractors on the decompiled code. + * Can be expanded to extract pretty much any desired data from the server software +4. pdba.exe -- analysis of PDB file for Minecraft bedrock edition Windows server binary +5. update3.js -- aggregate and finalize data, send to llm-services for further handling diff --git a/.github/helper-bot/disa.cpp b/.github/helper-bot/disa.cpp new file mode 100644 index 00000000..975ceca9 --- /dev/null +++ b/.github/helper-bot/disa.cpp @@ -0,0 +1,624 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include "disa.h" + +char *roData; +int roDataOffset; +int roDataEnd; +void loadRoData(std::string binFile) { + std::ifstream binStream(binFile, std::ios::binary); + if (!binStream.is_open()) { + std::cerr << "Failed to open file: " << binFile << std::endl; + return; + } + binStream.seekg(0, std::ios::end); + int size = binStream.tellg(); + binStream.seekg(0, std::ios::beg); + roData = new char[size]; + binStream.read(roData, size); + binStream.close(); + // End 9 bytes holds offset of the rodata section + std::string_view offsetStr(roData + size - 9, 8); + roDataOffset = hexStr2Int(offsetStr); + roDataEnd = roDataOffset + size; + fprintf(stderr, "Opened rodata file '%s', size: %d, offset: %d\n", binFile.c_str(), size, roDataOffset); +} + +// (A-Z, a-z, 0-9, symbols) +bool isValidAsciiChar(char c) { + return c >= 'A' && c <= '~'; +} +bool isAddressInRoData(unsigned int address) { + return address >= roDataOffset && address < roDataEnd; +} +bool isValidRoDataStrAddr(unsigned int address) { + if (!isAddressInRoData(address + 1)) { + return false; + } + auto bufferOffset = address - roDataOffset; + auto c = roData[bufferOffset]; + return isValidAsciiChar(c); +} + +// Get a null-terminated string from the rodata section +std::string getRoDataStringNT(unsigned int offset) { + if (!isValidRoDataStrAddr(offset)) { + return ""; + } + auto bufferOffset = offset - roDataOffset; + int len = 0; + while (isAddressInRoData(offset + len) && roData[bufferOffset + len] != '\0') { + len++; + } + return std::string(roData + bufferOffset, len); +} +float getRoDataFloat(unsigned int offset) { + if (!isAddressInRoData(offset)) { + return -0.0f; + } + auto bufferOffset = offset - roDataOffset; + return *(float *)(roData + bufferOffset); +} +std::string getRoDataHexDump(unsigned int offset, int len) { + if (!isAddressInRoData(offset)) { + return ""; + } + auto bufferOffset = offset - roDataOffset; + std::string hexDump; + for (int i = 0; i < len; i++) { + char c = roData[bufferOffset + i]; + char buffer[4]; + snprintf(buffer, 4, "%02x", c); + hexDump += buffer; + } + return hexDump; +} +std::string fnv64Hex(std::string_view str) { + unsigned long long hash = 0xcbf29ce484222325; + for (size_t i = 0; i < str.size(); i++) { + hash *= 0x100000001b3; + hash ^= str[i]; + } + char buffer[17]; + snprintf(buffer, 17, "%016llx", hash); + return "0x" + std::string(buffer); +} + +struct CurrentBlockData { + std::string blockName; + std::string blockClass; + unsigned int breakTimeAddr; + std::vector stateKeys; +}; +std::vector blockData; + +bool hasSeenBlockClass(const std::string_view &blockClass) { + for (auto &block : blockData) { + if (block.blockClass == blockClass) { + return true; + } + } + return false; +} + +bool contains(const std::vector &vec, const std::string &str) { + return std::find(vec.begin(), vec.end(), str) != vec.end(); +} + +void loadDisassembly(std::string filePath) { + // *uses AT&T syntax ; too late to change now + std::istream *disStream = &std::cin; + if (!filePath.empty()) { + disStream = new std::ifstream(filePath, std::ios::binary); + if (!((std::ifstream *)disStream)->is_open()) { + std::cerr << "Failed to open file: " << filePath << std::endl; + return; + } + } + + // 64KB buffer + const int bufferSize = 1024 * 64; + char buffer[bufferSize]; + std::string trackingBlock; + std::string trackingState; + bool isInGlobalBlock = false; + bool isInBlockRegistry = false; + + std::string inStateSerializer; + std::map> stateEntries; + + unsigned int lastLastLoadedAddress; + unsigned int lastLoadedAddress; + std::string lastLoadedAddressAbsMovStr; + + std::optional currentBlockData; + + std::vector seenBlockIds; + std::vector seenConstants; + // std::vector seenStates; + + Instruction instr; + + while (disStream->getline(buffer, bufferSize)) { + parseAttLine(buffer, instr); + if (instr.type == NO_INSTR) { + goto finish; + } + registerProcessInstruction(instr); + + // movabs $0x116b2c0, %rbx -> move the address to rbx + if (instr.type == MOVABS) { + std::string_view line(buffer); + size_t pos = line.find("$"); + size_t endPos = line.find(","); + if (pos != std::string::npos && endPos != std::string::npos) { + auto loadedStr = line.substr(pos + 1, endPos - pos - 1); + lastLoadedAddressAbsMovStr = loadedStr; + + // B1. if we are tracking a block, then print the constant + if (!trackingBlock.empty()) { + // if line includes '#', then split by the comment and get the comment + if (isValidRoDataStrAddr(lastLoadedAddress)) { + std::string str = getRoDataStringNT(lastLoadedAddress); + std::cout << "BlockID\t" << trackingBlock << "\t" << loadedStr << "\t" << str << std::endl; + seenBlockIds.push_back(trackingBlock); + } + } + } + } + + if (instr.type == CALL) { + std::string_view line(buffer); + size_t pos = line.find("registerBlock<"); + if (pos != std::string::npos) { + if (currentBlockData.has_value()) { + blockData.push_back(currentBlockData.value()); + currentBlockData.reset(); + } + + // class name is between "registerBlock<" and "," + size_t classStart = pos; + size_t classEnd = line.find(",", classStart); + auto blockClass = line.substr(classStart + 14, classEnd - classStart - 14); + if (trackingBlock.empty()) { + if (!hasSeenBlockClass(blockClass)) { + std::cerr << "? Unloaded Block registration: " << line << std::endl; + } + } else { + currentBlockData = CurrentBlockData{.blockName = trackingBlock, .blockClass = std::string(blockClass)}; + } + } + + size_t addStatePos = line.find("::addState(BlockState"); + if (addStatePos != std::string::npos) { + if (currentBlockData.has_value()) { + auto arg2 = registerGetArgInt(1); + if (arg2.symbolValue[0] && !contains(currentBlockData->stateKeys, arg2.symbolValue)) { + // VanillaStates::UpdateBit + auto symbol = std::string(arg2.symbolValue); + size_t statePos = symbol.find("::"); + if (statePos != std::string::npos) { + auto stateName = symbol.substr(statePos + 2); + if (stateName.find("::") != std::string::npos) { + stateName = stateName.substr(stateName.find("::") + 2); + } + currentBlockData->stateKeys.push_back(stateName); + } + } + } + } + + if (currentBlockData.has_value()) { + // 74061b8: callq 6bd9f60 + size_t destroyPos = line.find("setDestroyTime"); + if (destroyPos != std::string::npos) { + if (line.find(", float") != std::string::npos) { + // the last last loaded address is the address of the destroy time value + currentBlockData->breakTimeAddr = lastLastLoadedAddress; + } else { + // the last loaded address is the address of the destroy time value + currentBlockData->breakTimeAddr = lastLoadedAddress; + } + } + } + } + + if (instr.type == LEA) { + std::string_view line(buffer); + + if (instr.commentSymbolStart && instr.commentSymbolEnd) { + // there is a "# 86d4858 " comment + lastLastLoadedAddress = lastLoadedAddress; + lastLoadedAddress = instr.commentAddr; + + if (inStateSerializer.size() > 0) { + // we are interested in capturing all loaded constants inside the state serializer + if (isValidRoDataStrAddr(instr.commentAddr)) { + auto str = getRoDataStringNT(instr.commentAddr); + stateEntries[inStateSerializer].push_back(str); + } + } + + size_t constPos = line.find("SharedConstants::"); + if (constPos != std::string::npos) { + auto sharedName = line.substr(constPos + 17, line.size() - constPos - 18); + auto sharedNameStr = std::string(sharedName); + if (!contains(seenConstants, sharedNameStr)) { + seenConstants.push_back(sharedNameStr); + auto hexDump = getRoDataHexDump(instr.commentAddr, 32); + std::cout << "Const\t" << sharedName << "\t" << instr.commentAddr << "\t" << hexDump << std::endl; + } + } + + size_t pos = line.find("VanillaBlockTypeIds::"); + if (pos != std::string::npos) { + trackingBlock = line.substr(pos + 21, line.size() - pos - 22); + } + } + } else { + // B1. cont. Sometimes the movabs with hash is not after 2x lea ops, so we dump what we have and continue + if (!trackingBlock.empty() && instr.type != MOVABS) { + // If we've already seen the block, above check is not needed + if (!contains(seenBlockIds, trackingBlock)) { + // if line includes '#', then split by the comment and get the comment + if (isValidRoDataStrAddr(lastLoadedAddress)) { + auto str = getRoDataStringNT(lastLoadedAddress); + std::cout << "BlockID\t" << trackingBlock << "\t" << "UNK" << "\t" << str << std::endl; + } + } + } + // lea/mov are both used to load args before a call, so we can keep tracking if it's also a mov + if (instr.type != MOV) { + trackingBlock.clear(); + } + } + + // if a move over lea, we maybe loading block states + if (instr.type == MOV) { + std::string_view line(buffer); + // if line includes '#', then split by the comment and get the comment + if (instr.commentAddr) { + if (isInBlockRegistry || currentBlockData.has_value()) { + lastLastLoadedAddress = lastLoadedAddress; + lastLoadedAddress = instr.commentAddr; + } else if (isInGlobalBlock) { + size_t statesPos = line.find("VanillaStates::"); + size_t altStatePos = line.find("BuiltInBlockStates::"); + // State Registration + if ((statesPos != std::string::npos) || (altStatePos != std::string::npos)) { + // ensure there's no + offset in the symbol + if (line.find("+") != std::string::npos) { + goto finish; + } + auto states = statesPos != std::string::npos ? line.substr(statesPos + 15, line.size() - statesPos - 16) + : line.substr(altStatePos, line.size() - altStatePos - 1); + if (isValidRoDataStrAddr(lastLoadedAddress)) { + auto str = getRoDataStringNT(lastLoadedAddress); + auto computedHash = fnv64Hex(str); // lastLoadedAddressAbsMovStr can be optimized out + std::cout << "VanillaState\t" << states << "\t" << computedHash << "\t" << str << std::endl; + } else if (isValidRoDataStrAddr(lastLastLoadedAddress)) { + auto str = getRoDataStringNT(lastLastLoadedAddress); + auto computedHash = fnv64Hex(str); + std::cout << "VanillaState\t" << states << "\t" << computedHash << "\t" << str << std::endl; + } else { + // std::cout << "? NOT adding VanillaState\t" << states << " " << lastLoadedAddress << "\t" + // << lastLoadedAddressAbsMovStr << std::endl; + } + } + } + } + } + + // if buffer ends with a colon, then it's a new block + if (instr.isFunctionStart) { + std::string_view line(buffer); + trackingBlock.clear(); + // globals initialization block + if (line.find("_GLOBAL_") != std::string::npos) { + isInGlobalBlock = true; + } else { + isInGlobalBlock = false; + } + if (line.find("::registerBlocks") != std::string::npos) { + isInBlockRegistry = true; + } else { + isInBlockRegistry = false; + if (currentBlockData.has_value()) { + blockData.push_back(currentBlockData.value()); + currentBlockData.reset(); + } + } + + // 000000000715d070 (Tag const&, int&)>: + if (line.find("StateSerializationUtils::fromNBT<") != std::string::npos) { + auto pos = line.find("StateSerializationUtils::fromNBT<"); + auto end = line.find(">", pos); + auto substr = line.substr(pos + 33, end - pos - 33); + inStateSerializer = std::string(substr); + } else { + inStateSerializer.clear(); + } + } + finish: + ZeroMemory(buffer, bufferSize); + clearInstruction(instr); + } + + if (!filePath.empty()) { + ((std::ifstream *)disStream)->close(); + delete disStream; + } + // Print out the block data + for (auto &block : blockData) { + auto flt = getRoDataFloat(block.breakTimeAddr); + std::cout << "BlockData\t" << block.blockName << "\t" << block.blockClass << "\t" << flt << "\t"; + for (auto &state : block.stateKeys) { + std::cout << state << ","; + } + std::cout << std::endl; + } + + // Print out the state entries + for (auto &entry : stateEntries) { + std::cout << "StateEntry\t" << entry.first << "\t"; + for (auto &state : entry.second) { + std::cout << state << ","; + } + std::cout << std::endl; + } +} + +// STAGE 2 + +// StateHash -> integer data for this state (like number of variants) +std::map> stateVariantMap; + +void split4(std::string_view line, std::string &a, std::string &b, std::string &c, std::string &d) { + size_t pos = line.find("\t"); + if (pos != std::string::npos) { + a = std::string(line.substr(0, pos)); + size_t pos2 = line.find("\t", pos + 1); + if (pos2 != std::string::npos) { + b = std::string(line.substr(pos + 1, pos2 - pos - 1)); + size_t pos3 = line.find("\t", pos2 + 1); + if (pos3 != std::string::npos) { + c = std::string(line.substr(pos2 + 1, pos3 - pos2 - 1)); + d = std::string(line.substr(pos3 + 1, line.size() - pos3 - 1)); + } else { + c = std::string(line.substr(pos2 + 1, line.size() - pos2 - 1)); + } + } + } +} + +void loadStage1(std::string filePath) { + // load stage1 which is the output of above loadDisassembly function + std::ifstream stage1Stream(filePath, std::ios::binary); + if (!stage1Stream.is_open()) { + std::cerr << "Failed to open file: " << filePath << std::endl; + return; + } + // split by tabs + const int bufferSize = 1024 * 64; + char buffer[bufferSize]; + while (stage1Stream.getline(buffer, bufferSize)) { + std::string_view line(buffer); + size_t pos = line.find("\t"); + if (pos != std::string::npos) { + // VanillaState StateID StateHash StateName + // VanillaState BiteCounter 0x6bcbbe2ee1f42f72 bite_counter + // we are interested in the 2nd and 3rd columns + std::string name, id, hash, stateName; + split4(line, name, id, hash, stateName); + if (name == "VanillaState") { + // Strip the 0x prefix and leading zeros - not insignificant and disassembler may not include them + while (hash.size() > 0 && (hash[0] == '0' || hash[0] == 'x')) { + hash = hash.substr(1, hash.size() - 1); + } + stateVariantMap[hash] = {}; + } + } + } + fprintf(stderr, "Loaded %lu state variants from stage1\n", stateVariantMap.size()); +} + +std::map symbolMap; + +void loadStage4(std::string filePath) { + // load stage1 which is the output of above loadDisassembly function + std::ifstream stage4Stream(filePath, std::ios::binary); + if (!stage4Stream.is_open()) { + std::cerr << "Failed to open file: " << filePath << std::endl; + return; + } + // split by tabs + const int bufferSize = 1024 * 64; + char buffer[bufferSize]; + while (stage4Stream.getline(buffer, bufferSize)) { + std::string_view line(buffer); + size_t pos = line.find("\t"); + if (pos != std::string::npos) { + // WSYM Address SymbolNams + // WSYM 0x6bcbbe2ee1f42f72 BlockTrait::Something + std::string name, address, symbolName, _; + split4(line, name, address, symbolName, _); + if (name == "WSYM") { + uint64_t addressInt = hexStr2Int64(address); + symbolMap[addressInt] = symbolName; + } + } + } + fprintf(stderr, "Loaded %lu symbols from stage4\n", symbolMap.size()); +} + +bool haveSymbolForAddress(std::string_view addressStr) { + uint64_t addr = hexStr2Int64(addressStr); + return symbolMap.find(addr) != symbolMap.end(); +} + +std::string getSymbolForAddress(std::string_view address) { + return symbolMap[hexStr2Int64(address)]; +} + +void loadDisassembly2(std::string filePath) { + std::istream *disStream = &std::cin; + if (!filePath.empty()) { + disStream = new std::ifstream(filePath, std::ios::binary); + if (!((std::ifstream *)disStream)->is_open()) { + std::cerr << "Failed to open file: " << filePath << std::endl; + return; + } + } + + bool inStateVariant = false; + std::string currentHash; + + const int bufferSize = 1024 * 64; + char buffer[bufferSize]{0}; + + std::string trackingBlock; + // if trackingBlockFoundReg is > 0, we're tracking a block. We continue tracking for n instructions. + int trackingBlockFoundReg = 0; + + bool inMovInstruction = false; + // 140064b38: 48 c7 05 ad 7f 95 02 mov QWORD PTR [rip+0x2957fad],0x4 # 0x1429bcaf0 + while (disStream->getline(buffer, bufferSize)) { + if (STR_STARTS_WITH4(&buffer[36], "mov ")) { + inMovInstruction = true; + } else if (buffer[36] != ' ' && buffer[36] != '\0') { + // if the instruction is not a continuation of the previous instruction + inMovInstruction = false; + } + + // now we are looking for the state variants... first look for movabs + if (STR_STARTS_WITH4(&buffer[36], "movabs")) { + std::string_view line(buffer); + + for (auto &entry : stateVariantMap) { + size_t pos = line.find(entry.first); + if (pos != std::string::npos) { + // we found the state hash, now we need to find the number of variants + currentHash = entry.first; + } + } + } + + if (STR_STARTS_WITH4(&buffer[36], "lea ")) { + std::string_view line(buffer); + size_t addressPos = line.find(" # "); + if (addressPos != std::string::npos) { + auto addressStr = line.substr(addressPos + 5); + if (haveSymbolForAddress(addressStr)) { + auto symbol = getSymbolForAddress(addressStr); + size_t blockPos = symbol.find("VanillaBlockTypeIds::"); + if (blockPos != std::string::npos) { + trackingBlock = symbol.substr(blockPos + 21); + trackingBlockFoundReg = 0; + } + } + } + } + + if (STR_STARTS_WITH4(&buffer[36], "call")) { + std::string_view line(buffer); + auto addressIx = line.find("0x"); + if (addressIx != std::string::npos) { + auto addressStr = line.substr(addressIx + 2); + if (haveSymbolForAddress(addressStr)) { + auto symbol = getSymbolForAddress(addressStr); + if (trackingBlockFoundReg > 0) { + size_t traitPos = symbol.find("BlockTrait::"); + if (traitPos != std::string::npos) { + auto traitStr = symbol.substr(traitPos); + std::cout << "BlockTrait\t" << trackingBlock << "\t" << symbol << std::endl; + } + } + size_t pos = symbol.find("registerBlock<"); + if (pos != std::string::npos) { + trackingBlockFoundReg = trackingBlock.empty() ? 0 : 40; + } + } + } + } + + if (trackingBlockFoundReg) { + trackingBlockFoundReg--; + if (trackingBlockFoundReg == 0) + trackingBlock.clear(); + } + + // 140064b3f: 04 00 00 00 + if (inMovInstruction && buffer[36] == '\0' && currentHash.size() > 0) { + // this instruction is not an instruction but a continuation of the previous mov instruction + // this should contain the number of state variants + std::string_view line(buffer); + // split by the colon + size_t pos = line.find(":"); + if (pos != std::string::npos) { + auto hexStr = line.substr(pos + 1, line.size() - pos); + unsigned int value = hexStr2IntLE(hexStr); + stateVariantMap[currentHash].push_back(value); + // max is 3 entries + if (stateVariantMap[currentHash].size() == 3) { + currentHash.clear(); + } + } + } + + ZeroMemory(buffer, bufferSize); + } + + if (!filePath.empty()) { + ((std::ifstream *)disStream)->close(); + delete disStream; + } + + for (auto &entry : stateVariantMap) { + auto hash = entry.first; + // re-add the 0x and leading 0s + while (hash.size() < 16) { + hash = "0" + hash; + } + hash = "0x" + hash; + std::cout << "StateVariantData\t" << hash << "\t"; + for (auto &value : entry.second) { + std::cout << value << ","; + } + std::cout << std::endl; + } +} + +int main(int argc, char **argv) { + if (argc < 3) { + std::cerr << "Usage: disa -s1 [dis]" << std::endl; + std::cerr << "Usage: disa -s2 [dis]" << std::endl; + return 1; + } + std::string stage = argv[1]; + std::string file = argv[2]; + std::string disFile; + if (argc > 3) { + disFile = argv[3]; + } + std::cout << "Stage: " << stage << std::endl; + if (disFile.empty()) { + std::cerr << "(waiting for stdin)" << std::endl; + } + if (stage == "-s1") { + loadRoData(file); + loadDisassembly(disFile); + } else if (stage == "-s2") { + loadStage1(file); + // Not yet implemented: trait data extraction. This will require re-ordering and running stage4 before doing stage2. + // loadStage4("stage4.txt"); + loadDisassembly2(disFile); + } + std::cerr << "Done" << std::endl; + return 0; +} diff --git a/.github/helper-bot/disa.h b/.github/helper-bot/disa.h new file mode 100644 index 00000000..4ee81e41 --- /dev/null +++ b/.github/helper-bot/disa.h @@ -0,0 +1,637 @@ +#include +#include +#include +#include +#include + +// clang-format off +#define STR_STARTS_WITH2(str, other) ((str)[0] == other[0] && (str)[1] == other[1]) +#define STR_STARTS_WITH3(str, other) ((str)[0] == other[0] && (str)[1] == other[1] && str[2] == other[2]) +#define STR_STARTS_WITH4(s, o) ((s)[0] == o[0] && (s)[1] == o[1] && (s)[2] == o[2] && (s)[3] == o[3]) +#define STR_STARTS_WITH5(s, o) ((s)[0] == o[0] && (s)[1] == o[1] && (s)[2] == o[2] && (s)[3] == o[3] && (s)[4] == o[4]) +#define STR_STARTS_WITH6(s, o) ((s)[0] == o[0] && (s)[1] == o[1] && (s)[2] == o[2] && (s)[3] == o[3] && (s)[4] == o[4] && (s)[5] == o[5]) +#define STR_INCLUDES(haystack, needle) (haystack.find(needle) != std::string::npos) +// clang-format on + +typedef unsigned long long int u64; + +void ZeroMemory(char *buffer, int size) { + for (int i = 0; i < size; i++) { + buffer[i] = 0; + } +} + +void StringCopyInto(char *dest, const char *src) { + while (*src) { + *dest = *src; + dest++; + src++; + } + *dest = 0; +} +void StringCopyInto(char *dest, const char *src, int size, int max) { + size = size < max ? size : max; + ZeroMemory(dest, size); + for (int i = 0; i < size; i++) { + dest[i] = src[i]; + } + dest[size] = 0; +} +void StringStartsWith(std::string_view &str, std::string_view &prefix) { + if (str.size() < prefix.size()) { + return; + } + for (size_t i = 0; i < prefix.size(); i++) { + if (str[i] != prefix[i]) { + return; + } + } +} + +unsigned int hexStr2Int(const std::string_view &hexStr) { + const char *hexStrC = hexStr.data(); + unsigned int value = 0; + for (size_t i = 0; i < hexStr.size(); i++) { + char c = hexStrC[i]; + if (c >= '0' && c <= '9') { + value = (value << 4) + (c - '0'); + } else if (c >= 'a' && c <= 'f') { + value = (value << 4) + (c - 'a' + 10); + } else if (c >= 'A' && c <= 'F') { + value = (value << 4) + (c - 'A' + 10); + } + } + return value; +} + +unsigned int hexStr2IntLE(const std::string_view &hexStr) { + unsigned int value = hexStr2Int(hexStr); + unsigned int swapped = ((value >> 24) & 0xff) | // move byte 3 to byte 0 + ((value << 8) & 0xff0000) | // move byte 1 to byte 2 + ((value >> 8) & 0xff00) | // move byte 2 to byte 1 + ((value << 24) & 0xff000000); // byte 0 to byte 3 + return swapped; +} + +u64 hexStr2Int64(const std::string_view &hexStr) { + const char *hexStrC = hexStr.data(); + u64 value = 0; + for (size_t i = 0; i < hexStr.size(); i++) { + char c = hexStrC[i]; + if (c >= '0' && c <= '9') { + value = (value << 4) + (c - '0'); + } else if (c >= 'a' && c <= 'f') { + value = (value << 4) + (c - 'a' + 10); + } else if (c >= 'A' && c <= 'F') { + value = (value << 4) + (c - 'A' + 10); + } + } + return value; +} + +// INSTR PARSE +// 708c23b: lea 0x1647ede(%rip),%rsi # 86d4120 +enum InstructionType { NO_INSTR, MOVABS, MOV, LEA, CALL, OTHER, FUNCTION_START }; +struct Instruction { + InstructionType type; + bool isFunctionStart; + char *asciiAddressStart; + char *asciiAddressEnd; + char *asciiOpStart; + char *asciiOpEnd; + char *asciiOperandsStart; + char *asciiOperandsEnd; + char *asciiCommentStart; + char *asciiCommentEnd; + unsigned int commentAddr; + char *commentSymbolStart; + char *commentSymbolEnd; +}; +void clearInstruction(Instruction &instr) { + instr.type = NO_INSTR; + instr.isFunctionStart = false; + instr.asciiAddressStart = nullptr; + instr.asciiAddressEnd = nullptr; + instr.asciiOpStart = nullptr; + instr.asciiOpEnd = nullptr; + instr.asciiOperandsStart = nullptr; + instr.asciiOperandsEnd = nullptr; + instr.asciiCommentStart = nullptr; + instr.asciiCommentEnd = nullptr; + instr.commentAddr = 0; + instr.commentSymbolStart = nullptr; + instr.commentSymbolEnd = nullptr; +} +void parseAttLine(char *buffer, Instruction &instr) { + instr.asciiAddressStart = buffer; + if (buffer[0] == ' ') { + instr.isFunctionStart = false; + } else { + instr.isFunctionStart = true; + } + + if (instr.isFunctionStart) { + // 0000000002c44530 : + bool readingAddress = true; + bool readingSymbol = false; + int i = 0; + for (;; i++) { + auto c = buffer[i]; + if (c == '\0') + break; + if (c == ' ' || c == '\t') { + if (readingAddress) { + readingAddress = false; + readingSymbol = true; + instr.asciiAddressEnd = buffer + i; + instr.asciiOpStart = buffer + i + 1; + } + } + } + // op end here holds the symbol name + instr.asciiOpEnd = buffer + i; + // remove the trailing colon + instr.asciiOpEnd--; + } else { + bool readingAddress = true; + bool readingOp = false; + bool readingOperands = false; + bool readingComment = false; + for (int i = 0; true; i++) { + auto c = buffer[i]; + if (c == '\0') + break; + + if (readingAddress) { + for (int j = i; true; j++) { + auto c = buffer[j]; + if (c == '\0') + break; + if (c == ':') { + readingAddress = false; + readingOp = true; + instr.asciiAddressEnd = buffer + j; + i = j; + break; + } + } + } else if (readingOp) { + for (int j = i; true; j++) { + auto c = buffer[j]; + if (c == '\0') + break; + if (c == ' ' || c == '\t') { + if (instr.asciiOpStart) { + readingOp = false; + readingOperands = true; + instr.asciiOpEnd = buffer + j; + i = j; + break; + } + } else if (!instr.asciiOpStart) { + instr.asciiOpStart = buffer + j; + } + } + } else if (readingOperands) { + for (int j = i; true; j++) { + auto c = buffer[j]; + if (c == '#' || c == '\0') { + readingOperands = false; + readingComment = true; + instr.asciiOperandsEnd = buffer + j; + i = j; + break; + } else if (!(c == ' ' || c == '\t')) { + if (!instr.asciiOperandsStart) { + instr.asciiOperandsStart = buffer + j; + } + } + } + } else if (readingComment) { + for (int j = i; true; j++) { + auto c = buffer[j]; + if (c == '\0') { + readingComment = false; + instr.asciiCommentEnd = buffer + j; + i = j; + break; + } else if (!instr.asciiCommentStart) { + instr.asciiCommentStart = buffer + j; + } + } + } + } + } + + // Sanity check: make sure we have at least an op start and end (this also covers functions) + if (!instr.asciiOpStart || !instr.asciiOpEnd) { + instr.type = NO_INSTR; + instr.isFunctionStart = false; + return; + } + + if (instr.isFunctionStart) { + instr.type = FUNCTION_START; + return; + } + + auto op = instr.asciiOpStart; + if (op[0] == 'm' && op[1] == 'o' && op[2] == 'v' && op[3] == 'a' && op[4] == 'b' && op[5] == 's') { + instr.type = MOVABS; + } else if (op[0] == 'm' && op[1] == 'o' && op[2] == 'v') { + instr.type = MOV; + } else if (op[0] == 'l' && op[1] == 'e' && op[2] == 'a') { + instr.type = LEA; + } else if (op[0] == 'c' && op[1] == 'a' && op[2] == 'l' && op[3] == 'l') { + instr.type = CALL; + } else { + instr.type = OTHER; + } + + if (instr.asciiCommentStart && instr.asciiCommentEnd) { + // Comment Start: [ 86d4120 ] + // Comment End: [] + char *asciiCommentAddrStart = instr.asciiCommentStart + 1; + char *asciiCommentAddrEnd = nullptr; + char *asciiCommentSymbolStart = nullptr; + char *asciiCommentSymbolEnd = nullptr; + for (int i = 0; true; i++) { + auto c = asciiCommentAddrStart[i]; + if (c == '\0') + break; + if (c == '<' && !asciiCommentAddrEnd) { + asciiCommentAddrEnd = asciiCommentAddrStart + i; + asciiCommentSymbolStart = asciiCommentAddrStart + i + 1; + } else if (c == '>') { + asciiCommentSymbolEnd = asciiCommentAddrStart + i; + break; + } + } + if (asciiCommentAddrEnd && asciiCommentSymbolStart && asciiCommentSymbolEnd) { + instr.commentAddr = + hexStr2Int(std::string_view(asciiCommentAddrStart, asciiCommentAddrEnd - asciiCommentAddrStart)); + instr.commentSymbolStart = asciiCommentSymbolStart; + instr.commentSymbolEnd = asciiCommentSymbolEnd; + } + } +} +void instructionReadOperands(Instruction &instr, std::string_view &a, std::string_view &b, std::string_view &c) { + if (!instr.asciiOperandsStart || !instr.asciiOperandsEnd) { + return; + } + std::string_view operand = + std::string_view(instr.asciiOperandsStart, instr.asciiOperandsEnd - instr.asciiOperandsStart); + + // split by first comma + size_t commaPos1 = operand.find(','); + if (commaPos1 != std::string::npos) { + a = operand.substr(0, commaPos1); + std::string_view remaining = operand.substr(commaPos1 + 1); + + // split by second comma + size_t commaPos2 = remaining.find(','); + if (commaPos2 != std::string::npos) { + b = remaining.substr(0, commaPos2); + c = remaining.substr(commaPos2 + 1); + } else { + b = remaining; + } + } else { + a = operand; + } +} +// END INSTR PARSE + +/* +rax - register a extended +rbx - register b extended +rcx - register c extended +rdx - register d extended +rbp - register base pointer (start of stack) +rsp - register stack pointer (current location in stack, growing downwards) +rsi - register source index (source for data copies) +rdi - register destination index (destination for data copies) +*/ + +const int MAX_SYMBOL_SIZE = 63; +union RegisterVal { + enum RegisterDataType { RDTGeneric, RDTVanillaState, RDTBlockTypeID }; + u64 value; + char symbolValue[MAX_SYMBOL_SIZE + 1]; + + double doubleValue() { + return value == 0 ? -0 : *(double *)&value; + } +}; +struct RegisterState { + RegisterVal rax; // + RegisterVal rbx; // + RegisterVal rcx; // arg4 + RegisterVal rdx; // arg3 + RegisterVal rbp; // start of stack + RegisterVal rsp; // current location in stack, growing downwards + RegisterVal rsi; // register source index (source for data copies) ; arg2 + RegisterVal rdi; // register destination index (destination for data copies) ; arg1 + RegisterVal r8; + RegisterVal r9; + RegisterVal r10; + RegisterVal r11; + RegisterVal r12; + RegisterVal r13; + RegisterVal r14; + RegisterVal r15; + RegisterVal rip; + RegisterVal rflags; + RegisterVal xmm0; + RegisterVal xmm1; + RegisterVal xmm2; + RegisterVal xmm3; +}; +RegisterState g_registerState; +enum Register { + REG_UNKNOWN, + RAX, + RBX, + RCX, + RDX, + RBP, + RSP, + RSI, + RDI, + R8, + R9, + R10, + R11, + R12, + R13, + R14, + R15, + RIP, + RFLAGS, + XMM0, + XMM1, + XMM2, + XMM3 +}; + +void registerClearState() { + g_registerState = RegisterState{}; +} + +Register registerGetType(std::string_view str) { + // in AT&T syntax, registers are prefixed with % + std::string_view reg = str[0] == '%' ? str.substr(1) : str; + // std::cout << "Reading register: [" << reg << "]" << std::endl; + // clang-format off + if (STR_STARTS_WITH3(reg, "rax")) return RAX; + if (STR_STARTS_WITH3(reg, "rbx")) return RBX; + if (STR_STARTS_WITH3(reg, "rcx")) return RCX; + if (STR_STARTS_WITH3(reg, "rdx")) return RDX; + if (STR_STARTS_WITH3(reg, "rbp")) return RBP; + if (STR_STARTS_WITH3(reg, "rsp")) return RSP; + if (STR_STARTS_WITH3(reg, "rsi")) return RSI; + if (STR_STARTS_WITH3(reg, "rdi")) return RDI; + if (STR_STARTS_WITH2(reg, "r8")) return R8; + if (STR_STARTS_WITH2(reg, "r9")) return R9; + if (STR_STARTS_WITH3(reg, "r10")) return R10; + if (STR_STARTS_WITH3(reg, "r11")) return R11; + if (STR_STARTS_WITH3(reg, "r12")) return R12; + if (STR_STARTS_WITH3(reg, "r13")) return R13; + if (STR_STARTS_WITH3(reg, "r14")) return R14; + if (STR_STARTS_WITH3(reg, "r15")) return R15; + if (STR_STARTS_WITH3(reg, "rip")) return RIP; + if (STR_STARTS_WITH5(reg, "rflag")) return RFLAGS; + if (STR_STARTS_WITH4(reg, "xmm0")) return XMM0; + if (STR_STARTS_WITH4(reg, "xmm1")) return XMM1; + if (STR_STARTS_WITH4(reg, "xmm2")) return XMM2; + if (STR_STARTS_WITH4(reg, "xmm3")) return XMM3; + // clang-format on + return REG_UNKNOWN; +} + +// The first four integer or pointer parameters are passed in the first four general-purpose registers, rdi, rsi, rdx, +// and rcx. The first four floating-point parameters are passed in the first four SSE registers, xmm0-xmm3. +void registerSwap(RegisterVal &a, RegisterVal &b) { + RegisterVal temp = a; + a = b; + b = temp; +} + +RegisterVal registerGetArgFloat(int index) { + switch (index) { + case 0: + return g_registerState.xmm0; + case 1: + return g_registerState.xmm1; + case 2: + return g_registerState.xmm2; + case 3: + return g_registerState.xmm3; + default: + return RegisterVal{}; + } +} + +RegisterVal registerGetArgInt(int index) { + switch (index) { + case 0: + return g_registerState.rdi; + case 1: + return g_registerState.rsi; + case 2: + return g_registerState.rdx; + case 3: + return g_registerState.rcx; + default: + return RegisterVal{}; + } +} + +#define REG_CASE(U, L) \ + case U: { \ + /*std::cout << "Setting " << #L << " to " << value << std::endl;*/ \ + g_registerState.L.value = value; \ + if (hasCommentSym) \ + StringCopyInto(g_registerState.L.symbolValue, comment.data(), comment.size(), MAX_SYMBOL_SIZE); \ + return &g_registerState.L; \ + } + +RegisterVal *registerSetVal(Register ®, u64 value, bool hasCommentSym, std::string_view comment) { + // std::cout << "Setting Register: " << reg << " to " << value << std::endl; + switch (reg) { + REG_CASE(RAX, rax) + REG_CASE(RBX, rbx) + REG_CASE(RCX, rcx) + REG_CASE(RDX, rdx) + REG_CASE(RBP, rbp) + REG_CASE(RSP, rsp) + REG_CASE(RSI, rsi) + REG_CASE(RDI, rdi) + REG_CASE(R8, r8) + REG_CASE(R9, r9) + REG_CASE(R10, r10) + REG_CASE(R11, r11) + REG_CASE(R12, r12) + REG_CASE(R13, r13) + REG_CASE(R14, r14) + REG_CASE(R15, r15) + REG_CASE(RIP, rip) + REG_CASE(RFLAGS, rflags) + REG_CASE(XMM0, xmm0) + REG_CASE(XMM1, xmm1) + REG_CASE(XMM2, xmm2) + REG_CASE(XMM3, xmm3) + default: + break; + } + return nullptr; +} + +#undef REG_CASE + +void registerCopy(RegisterVal &a, RegisterVal &intoB) { + // copy the value + intoB.value = a.value; + // copy the symbol + StringCopyInto(intoB.symbolValue, a.symbolValue, MAX_SYMBOL_SIZE, MAX_SYMBOL_SIZE); +} + +// clang-format off +#define REG_MOVE_CASE(UFROM, FROM) \ + case UFROM: \ + switch (intoRegSlot) { \ + case RAX: registerCopy(g_registerState.FROM, g_registerState.rax); return &g_registerState.rax; \ + case RBX: registerCopy(g_registerState.FROM, g_registerState.rbx); return &g_registerState.rbx; \ + case RCX: registerCopy(g_registerState.FROM, g_registerState.rcx); return &g_registerState.rcx; \ + case RDX: registerCopy(g_registerState.FROM, g_registerState.rdx); return &g_registerState.rdx; \ + case RBP: registerCopy(g_registerState.FROM, g_registerState.rbp); return &g_registerState.rbp; \ + case RSP: registerCopy(g_registerState.FROM, g_registerState.rsp); return &g_registerState.rsp; \ + case RSI: registerCopy(g_registerState.FROM, g_registerState.rsi); return &g_registerState.rsi; \ + case RDI: registerCopy(g_registerState.FROM, g_registerState.rdi); return &g_registerState.rdi; \ + case R8: registerCopy(g_registerState.FROM, g_registerState.r8); return &g_registerState.r8; \ + case R9: registerCopy(g_registerState.FROM, g_registerState.r9); return &g_registerState.r9; \ + case R10: registerCopy(g_registerState.FROM, g_registerState.r10); return &g_registerState.r10; \ + case R11: registerCopy(g_registerState.FROM, g_registerState.r11); return &g_registerState.r11; \ + case R12: registerCopy(g_registerState.FROM, g_registerState.r12); return &g_registerState.r12; \ + case R13: registerCopy(g_registerState.FROM, g_registerState.r13); return &g_registerState.r13; \ + case R14: registerCopy(g_registerState.FROM, g_registerState.r14); return &g_registerState.r14; \ + case R15: registerCopy(g_registerState.FROM, g_registerState.r15); return &g_registerState.r15; \ + case RIP: registerCopy(g_registerState.FROM, g_registerState.rip); return &g_registerState.rip; \ + case RFLAGS: registerCopy(g_registerState.FROM, g_registerState.rflags); return &g_registerState.rflags; \ + case XMM0: registerCopy(g_registerState.FROM, g_registerState.xmm0); return &g_registerState.xmm0; \ + case XMM1: registerCopy(g_registerState.FROM, g_registerState.xmm1); return &g_registerState.xmm1; \ + case XMM2: registerCopy(g_registerState.FROM, g_registerState.xmm2); return &g_registerState.xmm2; \ + case XMM3: registerCopy(g_registerState.FROM, g_registerState.xmm3); return &g_registerState.xmm3; \ + default: break; \ + } \ + break; +// clang-format on + +// Register holds an integer. It does not hold the value of the register itself. No recursion! +RegisterVal *registerMove(Register &fromRegSlot, Register &intoRegSlot) { + // printf("Moving Register: %d into Register: %d\n", fromRegSlot, intoRegSlot); + switch (fromRegSlot) { + REG_MOVE_CASE(RAX, rax) + REG_MOVE_CASE(RBX, rbx) + REG_MOVE_CASE(RCX, rcx) + REG_MOVE_CASE(RDX, rdx) + REG_MOVE_CASE(RBP, rbp) + REG_MOVE_CASE(RSP, rsp) + REG_MOVE_CASE(RSI, rsi) + REG_MOVE_CASE(RDI, rdi) + REG_MOVE_CASE(R8, r8) + REG_MOVE_CASE(R9, r9) + REG_MOVE_CASE(R10, r10) + REG_MOVE_CASE(R11, r11) + REG_MOVE_CASE(R12, r12) + REG_MOVE_CASE(R13, r13) + REG_MOVE_CASE(R14, r14) + REG_MOVE_CASE(R15, r15) + REG_MOVE_CASE(RIP, rip) + REG_MOVE_CASE(RFLAGS, rflags) + REG_MOVE_CASE(XMM0, xmm0) + REG_MOVE_CASE(XMM1, xmm1) + REG_MOVE_CASE(XMM2, xmm2) + REG_MOVE_CASE(XMM3, xmm3) + default: + break; + } + return nullptr; +} + +// We are really only interested in MOV / MOVABS, and LEA instructions. +void registerProcessInstruction(Instruction &instr) { + if (instr.type == FUNCTION_START) { + registerClearState(); + return; + } else if (instr.type == NO_INSTR) { + return; + } + + std::string_view operand1, operand2, operand3; + instructionReadOperands(instr, operand1, operand2, operand3); + + std::string_view commentSymbol; + bool hasCommentSymbol = instr.commentSymbolStart && instr.commentSymbolEnd; + if (hasCommentSymbol) { + commentSymbol = std::string_view(instr.commentSymbolStart, instr.commentSymbolEnd - instr.commentSymbolStart); + } + + switch (instr.type) { + case MOVABS: { + // movabs $0x116b2c0, %rbx + // a1 is the value, a2 is the register + Register a1 = registerGetType(operand2); + u64 a2 = hexStr2Int64(operand1); + registerSetVal(a1, a2, hasCommentSymbol, commentSymbol); + break; + } + case MOV: { + // mov %rdi,%rax + // a1 is the source register, a2 is the destination register + Register a1 = registerGetType(operand1); + Register a2 = registerGetType(operand2); + RegisterVal *reg = registerMove(a1, a2); + if (hasCommentSymbol && reg) { + StringCopyInto(reg->symbolValue, commentSymbol.data(), commentSymbol.size(), MAX_SYMBOL_SIZE); + } + break; + } + case LEA: { + // lea 0x1647ede(%rip),%rsi # 86d4120 + // a1 is the address, a2 is the register. The address typically is resolved to a symbol by objdump disassembler, so + // we can use it instead of a relative address. + Register intoReg = registerGetType(operand2); + RegisterVal *rv = registerSetVal(intoReg, instr.commentAddr, hasCommentSymbol, commentSymbol); + break; + } + case CALL: + case OTHER: + break; + case FUNCTION_START: + break; + default: + break; + } +} + +void registerDumpCallArgs() { + auto arg1 = registerGetArgInt(0); + auto arg2 = registerGetArgInt(1); + auto arg3 = registerGetArgInt(2); + auto fArg1 = registerGetArgFloat(0); + auto fArg2 = registerGetArgFloat(1); + auto fArg3 = registerGetArgFloat(2); + // clang-format off + fprintf( + stderr, + "Args: iArg1: %lld (%s), iArg2: %lld (%s), iArg3: %lld (%s) ; fArg1: %f (%s), fArg2: %f (%s), fArg3: %f (%s)\n", + arg1.value, arg1.symbolValue, + arg2.value, arg2.symbolValue, + arg3.value, arg3.symbolValue, + fArg1.doubleValue(), fArg1.symbolValue, + fArg2.doubleValue(), fArg2.symbolValue, + fArg3.doubleValue(), fArg3.symbolValue + ); + // clang-format on +} \ No newline at end of file diff --git a/.github/helper-bot/index.js b/.github/helper-bot/index.js index 8dffc6c0..152f82e3 100644 --- a/.github/helper-bot/index.js +++ b/.github/helper-bot/index.js @@ -1,8 +1,10 @@ // Automatic version update checker for Minecraft bedrock edition. const fs = require('fs') +const { join } = require('path') const cp = require('child_process') +const core = require('@actions/core') const helper = require('gh-helpers')() -const latestVesionEndpoint = 'https://itunes.apple.com/lookup?bundleId=com.mojang.minecraftpe&time=' + Date.now() +const latestVersionEndpoint = 'https://itunes.apple.com/lookup?bundleId=com.mojang.minecraftpe&time=' + Date.now() const changelogURL = 'https://feedback.minecraft.net/hc/en-us/sections/360001186971-Release-Changelogs' // Relevant infomation for us is: @@ -12,7 +14,6 @@ const changelogURL = 'https://feedback.minecraft.net/hc/en-us/sections/360001186 function buildFirstIssue (title, result, externalPatches) { let commitData = '' - let protocolVersion = '?' const date = new Date(result.currentVersionReleaseDate).toUTCString() for (const name in externalPatches) { @@ -24,7 +25,6 @@ function buildFirstIssue (title, result, externalPatches) { if (diff) commitData += `\n**[See the diff between *${result.currentVersionReleaseDate}* and now](${diff})**\n` else commitData += '\n(No changes so far)\n' } - try { protocolVersion = getProtocolVersion() } catch (e) { console.log(e) } return { title, @@ -39,16 +39,15 @@ A new Minecraft Bedrock version is available (as of ${date}), version **${result ${commitData} ## Protocol Details -(I will close this issue automatically if "${result.version}" is added to index.d.ts on "master" and there are no X's below) - - + + +
Name${result.version}
Protocol ID${protocolVersion}
+*I'll try to close this issue automatically if the protocol version didn't change. If the protocol version did change, the automatic update system will try to complete an update to minecraft-data and bedrock-protocol and if successful it will auto close this issue.* + ----- 🤖 I am a bot, I check for updates every 2 hours without a trigger. You can close this issue to prevent any further updates. @@ -56,11 +55,10 @@ ${commitData} } } -function getCommitsInRepo (repo, containing, since) { +async function getCommitsInRepo (repo, containing, since) { const endpoint = `https://api.github.com/repos/${repo}/commits` console.log('Getting', endpoint) - cp.execSync(`curl -L ${endpoint} -o commits.json`, { stdio: 'inherit', shell: true }) - const commits = JSON.parse(fs.readFileSync('./commits.json', 'utf-8')) + const commits = await fetch(endpoint).then(res => res.json()) const relevant = [] for (const commit of commits) { if (commit.commit.message.includes(containing)) { @@ -74,29 +72,20 @@ function getCommitsInRepo (repo, containing, since) { if (commits.length) { const head = commits[0].sha const tail = commits[commits.length - 1].sha - return [relevant, `https://github.com/${repo}/compare/${tail}..${head}`] + return [relevant, `https://github.com/${repo}/compare/${tail}..${head}`] } } return [relevant] } -function getProtocolVersion () { - if (!fs.existsSync('./ProtocolInfo.php')) cp.execSync('curl -LO https://raw.githubusercontent.com/pmmp/PocketMine-MP/stable/src/pocketmine/network/mcpe/protocol/ProtocolInfo.php', { stdio: 'inherit', shell: true }) - const currentApi = fs.readFileSync('./ProtocolInfo.php', 'utf-8') - const [, latestProtocolVersion] = currentApi.match(/public const CURRENT_PROTOCOL = (\d+);/) - return latestProtocolVersion -} - async function fetchLatest () { - if (!fs.existsSync('./results.json')) cp.execSync(`curl -L "${latestVesionEndpoint}" -o results.json`, { stdio: 'inherit', shell: true }) - const json = require('./results.json') + // curl -L "https://itunes.apple.com/lookup?bundleId=com.mojang.minecraftpe" -o results.json + const json = await fetch(latestVersionEndpoint).then(res => res.json()) const result = json.results[0] - // console.log(json) - if (!fs.existsSync('./index.d.ts')) cp.execSync('curl -LO https://raw.githubusercontent.com/PrismarineJS/bedrock-protocol/master/index.d.ts', { stdio: 'inherit', shell: true }) - const currentApi = fs.readFileSync('./index.d.ts', 'utf-8') - const supportedVersions = currentApi.match(/type Version = ([^\n]+)/)[1].replace(/\||'/g, ' ').split(' ').map(k => k.trim()).filter(k => k.length) - console.log(supportedVersions) + const currentTypes = fs.readFileSync(join(__dirname, '../../index.d.ts'), 'utf-8') + const supportedVersions = currentTypes.match(/type Version = ([^\n]+)/)[1].replace(/\||'/g, ' ').split(' ').map(k => k.trim()).filter(k => k.length) + console.log('Supported versions', supportedVersions) let { version, currentVersionReleaseDate, releaseNotes } = result console.log(version, currentVersionReleaseDate, releaseNotes) @@ -112,7 +101,6 @@ async function fetchLatest () { return } - if (issueStatus.isClosed) { // We already made an issue, but someone else already closed it, don't do anything else console.log('I already made an issue, but it was closed') @@ -121,15 +109,20 @@ async function fetchLatest () { version = version.replace('.0', '') const issuePayload = buildFirstIssue(title, result, { - PocketMine: getCommitsInRepo('pmmp/PocketMine-MP', version, currentVersionReleaseDate), - gophertunnel: getCommitsInRepo('Sandertv/gophertunnel', version, currentVersionReleaseDate), - CloudburstMC: getCommitsInRepo('CloudburstMC/Protocol', version, currentVersionReleaseDate) + PocketMine: await getCommitsInRepo('pmmp/PocketMine-MP', version, currentVersionReleaseDate), + gophertunnel: await getCommitsInRepo('Sandertv/gophertunnel', version, currentVersionReleaseDate), + CloudburstMC: await getCommitsInRepo('CloudburstMC/Protocol', version, currentVersionReleaseDate) }) if (issueStatus.isOpen) { - helper.updateIssue(issueStatus.id, issuePayload) + await helper.updateIssue(issueStatus.id, issuePayload) + // TEMP TEST + core.setOutput('updateVersion', version) + core.setOutput('issueNumber', issueStatus.id) } else { - helper.createIssue(issuePayload) + const issue = await helper.createIssue(issuePayload) + core.setOutput('updateVersion', version) + core.setOutput('issueNumber', issue.number) } fs.writeFileSync('./issue.md', issuePayload.body) diff --git a/.github/helper-bot/package.json b/.github/helper-bot/package.json new file mode 100644 index 00000000..ae068535 --- /dev/null +++ b/.github/helper-bot/package.json @@ -0,0 +1,10 @@ +{ + "description": "Minecraft Bedrock automatic update system", + "scripts": { + "test": "standard", + "fix": "standard --fix" + }, + "dependencies": { + "gh-helpers": "^0.2.1" + } +} diff --git a/.github/helper-bot/pdba.cpp b/.github/helper-bot/pdba.cpp new file mode 100644 index 00000000..a821f5cb --- /dev/null +++ b/.github/helper-bot/pdba.cpp @@ -0,0 +1,146 @@ +#include +#include +#include +#include +#include + +void readVanillaState(std::string &demangled) { + auto vanillaStatesPos = demangled.find("VanillaStates::"); + if (vanillaStatesPos == std::string::npos) { + return; + } + auto vanillaStateName = demangled.substr(vanillaStatesPos + 15); + if (demangled.find("Variant") != std::string::npos) { + std::cout << "VST\t" << vanillaStateName << "\t" << "int" << std::endl; + } else if (demangled.find("Variant") != std::string::npos) { + std::cout << "VST\t" << vanillaStateName << "\t" << "bool" << std::endl; + } else { + // Capture what's in the Variant<...> + auto variantPos = demangled.find("Variant<"); + if (variantPos != std::string::npos) { + auto variantEndPos = demangled.find(">", variantPos); + auto variantType = demangled.substr(variantPos + 8, variantEndPos - variantPos - 8); + std::cout << "VST\t" << vanillaStateName << "\t" << "string" << "\t" << variantType << std::endl; + } else { + std::cout << "VST\t" << vanillaStateName << "\t" << "string" << std::endl; + } + } +} + +void readConstant(std::string &demangled) { + // enum CodeBuilder::ProtocolVersion const SharedConstants::CodeBuilderProtocolVersion + auto sharedPos = demangled.find("SharedConstants::"); + if (sharedPos == std::string::npos) { + return; + } + auto sharedName = demangled.substr(sharedPos + 17); + auto type = demangled.substr(0, sharedPos - 1); + std::cout << "SCT\t" << sharedName << "\t" << type << std::endl; +} + +unsigned int parseInt(const std::string &str) { + unsigned int result = 0; + for (auto c : str) { + result = result * 10 + (c - '0'); + } + return result; +} + +std::string int2hex(uint64_t i) { + std::stringstream stream; + stream << std::hex << i; + return stream.str(); +} + +uint64_t hex2int(const std::string &hex) { + uint64_t result = 0; + for (auto c : hex) { + result = result * 16 + (c >= '0' && c <= '9' ? c - '0' : c - 'a' + 10); + } + return result; +} + +int find(std::string &what, std::string subStr) { + for (int i = 0; i < what.size(); i++) { + if (what[i] == subStr[0]) { + bool found = true; + for (int j = 1; j < subStr.size(); j++) { + if (what[i + j] != subStr[j]) { + found = false; + break; + } + } + if (found) { + return i; + } + } + } + return -1; +} + +#define STR_INCLUDES(haystack, needle) (haystack.find(needle) != std::string::npos) + +void loadDump(uint64_t textOffset = 0x140001000, uint64_t relocOffset = 0x142bbd000) { + uint64_t newOffset = relocOffset + 0x4A0000; + std::string line; + std::string readingBlockTrait; + + while (std::getline(std::cin, line)) { + if (readingBlockTrait.size() > 0) { + // flags = function, addr = 0001:30564736 + auto symInfo = line.find("addr = "); + if (symInfo != std::string::npos) { + auto addr = line.find(":"); + if (addr != std::string::npos) { + auto sectionId = line[addr - 1]; + auto addrStr = line.substr(addr + 1); + if (sectionId == '1') { // .text + uint64_t addrInt = parseInt(addrStr) + textOffset; + std::cout << "WSYM\t" << int2hex(addrInt) << "\t" << readingBlockTrait << std::endl; + } else if (sectionId == '3') { // .data + uint64_t addrInt = parseInt(addrStr) + newOffset; + std::cout << "WSYM\t" << int2hex(addrInt) << "\t" << readingBlockTrait << std::endl; + } + } + } + readingBlockTrait = ""; + } + + auto pos = line.find("`?"); + if (pos != std::string::npos) { + std::string mangledName = line.substr(pos + 1, line.size() - pos - 2); + std::string demangled = llvm::demangle(mangledName); + + if (mangledName.find("VanillaStates") != std::string::npos) { + readVanillaState(demangled); + } else if (mangledName.find("SharedConstants") != std::string::npos) { + readConstant(demangled); + } + if (STR_INCLUDES(line, "S_PUB32")) { + // Record only on the interesting things + if (STR_INCLUDES(line, "BlockTrait") || STR_INCLUDES(line, "VanillaBlockTypeIds") || + STR_INCLUDES(line, "registerBlock")) { + readingBlockTrait = demangled; + } + } + } + } + printf("Done\n"); +} + +// 0 .text 022692fc 0000000140001000 0000000140001000 00000400 2**4 + +int main(int argc, char **argv) { + if (argc < 3) { + std::cerr << "Usage: " << argv[0] << " " << std::endl; + return 1; + } + std::string textOffsetStr = argv[1]; + std::string relocOffsetStr = argv[2]; + std::cerr << "textOffset: " << textOffsetStr << std::endl; + std::cerr << "relocOffset: " << relocOffsetStr << std::endl; + uint64_t textOffset = std::stol(textOffsetStr, nullptr, 16); + uint64_t relocOffset = std::stol(relocOffsetStr, nullptr, 16); + loadDump(textOffset, relocOffset); + return 0; +} diff --git a/.github/helper-bot/serverScript.js b/.github/helper-bot/serverScript.js new file mode 100644 index 00000000..28c5e682 --- /dev/null +++ b/.github/helper-bot/serverScript.js @@ -0,0 +1,34 @@ +// Minecraft Bedrock Edition behavior pack script to extract block data. +// Based off https://github.com/Alemiz112/BedrockUtils/tree/master/BlockPaletteDumperAddon +import { + BlockStates, + BlockTypes, + BlockPermutation +} from '@minecraft/server' + +const data = { + blocks: [], + blockProperties: BlockStates.getAll() +} + +const blocks = BlockTypes.getAll() +for (let i = 0; i < blocks.length; i++) { + const permutation = BlockPermutation.resolve(blocks[i].id) + const defaultPermutation = permutation.getAllStates() + const blockData = { + name: blocks[i].id, + defaultState: defaultPermutation, + stateTypes: Object.fromEntries(Object.keys(defaultPermutation).map(e => [e, typeof e])), + stateValues: {} + } + const stateNames = Object.keys(defaultPermutation) + for (let j = 0; j < stateNames.length; j++) { + const stateName = stateNames[j] + const state = BlockStates.get(stateName) + const validValues = state.validValues + blockData.stateValues[stateName] = validValues + } + data.blocks.push(blockData) +} + +console.warn('' + JSON.stringify(data) + '') diff --git a/.github/helper-bot/update1.js b/.github/helper-bot/update1.js new file mode 100644 index 00000000..0ce87ba7 --- /dev/null +++ b/.github/helper-bot/update1.js @@ -0,0 +1,227 @@ +/* eslint-disable no-var, no-extend-native */ +const fs = require('fs') +const path = require('path') +const mcData = require('minecraft-data') +const latestSupportedProtocol = mcData.versions.bedrock[0].version +const bedrock = require('bedrock-protocol') +const bedrockServer = require('minecraft-bedrock-server') +let core +/** @type {import('gh-helpers').GithubHelper} */ +let github +if (process.env.CI) { + core = require('@actions/core') + github = require('gh-helpers')() +} else { + globalThis.isMocha = true + core = { setOutput: (name, value) => console.log(name, value) } + github = require('gh-helpers')() + github.getIssue = () => ({ body: '(Demo)' }) +} + +BigInt.prototype.toJSON = function () { + return this.toString() +} + +async function tryConnect (opts) { + const client = bedrock.createClient({ + host: 'localhost', + port: 19130, + username: 'test', + offline: true + }) + client.on('connect_allowed', () => { Object.assign(client.options, opts) }) + + const collected = {} + const forCollection = ['start_game', 'available_commands'] + + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + if (Object.keys(collected).length !== forCollection.length) { + reject(Error('Unable to collect all packets')) + } + }, 1000 * 60 * 2) + + function done (data) { + client.close() + clearTimeout(timeout) + resolve(data) + } + + for (const packet of forCollection) { + console.log('Waiting for', packet) + client.once(packet, (data) => { + console.log('Received', packet) + collected[packet] = data + fs.writeFileSync(path.join(__dirname, '/collected.json'), JSON.stringify(collected, null, 2)) + if (Object.keys(collected).length === forCollection.length) { + done(collected) + } + }) + } + }) +} + +async function main (inputUpdateVer, inputIssueNo) { + const issue = await github.getIssue(inputIssueNo) + const latestServers = await bedrockServer.getLatestVersions() + console.log('Issue data', issue) + let updatedBody = issue.body + const serverVersion = latestServers.linux.version3 + if (serverVersion !== inputUpdateVer) { + updatedBody = updatedBody.replace('', `Server version${serverVersion}`) + } + const server = await bedrockServer.prepare(serverVersion, { root: __dirname }) + const serverPath = server.path + console.log('Server version', serverVersion, 'Server path', serverPath) + core.setOutput('serverVersion', serverVersion) + core.setOutput('serverPath', serverPath) + core.setOutput('serverBin', serverPath + '/bedrock_server_symbols.debug') + await server.clearBehaviorPacks() + const handle = await server.startAndWaitReady(60000) + await new Promise((resolve) => setTimeout(resolve, 2000)) + const pong = await bedrock.ping({ host: '127.0.0.1', port: 19130, timeout: 4000 }) + updatedBody = updatedBody.replace('', `Protocol ID${pong.protocol} (${pong.version})`) + try { + await tryConnect({ protocolVersion: pong.protocol }) + updatedBody = updatedBody.replace('', 'Partly Already CompatibleYes') + } catch (e) { + console.error(e) + updatedBody = updatedBody.replace('', 'Partly Already CompatibleNO') + } + fs.writeFileSync(path.join(__dirname, '/updatedBody.md'), updatedBody) + await github.updateIssue(inputIssueNo, { body: updatedBody }) + console.log('Updated issue body', inputIssueNo, updatedBody) + handle.kill() + + // Check if protocol version has changed + if (pong.protocol === latestSupportedProtocol) { + console.log('Protocol version has not changed') + // Close the github issue + await github.close(inputIssueNo, 'Protocol version has not changed, assuming no compatibility issues.') + core.setOutput('needsUpdate', false) + return + } else { + core.setOutput('needsUpdate', true) + core.setOutput('protocolVersion', pong.protocol) + } + + // If the protocol version was changed, start server again, but with a behavior pack + console.log('⚒️ Re-running Bedrock server with extractor behavior pack') + + // First, determine the latest script version + injectPack(server, '1.0.0-beta') + const handle2 = await server.startAndWaitReady(10000) + const scriptVersion = await collectScriptVersion(handle2) + handle2.kill() + + // Re-run the server with the new script version + injectPack(server, scriptVersion) + const handle3 = await server.startAndWaitReady(10000) + const blockData = await collectDump(handle3) + fs.writeFileSync(path.join(__dirname, '/collectedBlockData.json'), blockData) + handle3.kill() + + console.log('✅ Finished working with Linux server binary') + console.log('Working now on Windows') + const winPath = serverPath.replace('bds-', 'bds-win-') + await bedrockServer.downloadServer(latestServers.windows.version3, { path: winPath, platform: 'windows' }) + core.setOutput('serverWinPath', winPath) + core.setOutput('serverWinBin', winPath + '/bedrock_server.exe') + core.setOutput('serverWinPdb', winPath + '/bedrock_server.pdb') + console.log('✅ Finished working with Windows server binary') +} + +main(process.env.UPDATE_VERSION, process.env.ISSUE_NUMBER) +// if (!process.env.CI) main('1.20.73', 0) + +function collectScriptVersion (handle, timeout = 1000 * 20) { + // The scripting API doesn't support semantic versioning with tilde or caret operators + // so we need to extract the version from the server log + let onceTimer + let onceDone + function onceWithDelay (fn, delay) { + if (onceDone) return + clearTimeout(onceTimer) + onceTimer = setTimeout(() => { + fn() + onceDone = true + }, delay) + } + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + reject(Error('Timeout while waiting for dump')) + }, timeout) + let total = '' + function process (log) { + const data = log.toString() + total += data + for (const line of total.split('\n')) { + if (line.includes('@minecraft/server -')) { + onceWithDelay(() => { + const scriptVersion = line.split('@minecraft/server -')[1].trim() + console.log('Latest @minecraft/server version is', scriptVersion) + clearTimeout(timer) + resolve(scriptVersion) + handle.stdout.off('data', process) + }, 500) + } + } + } + handle.stdout.on('data', process) + }) +} + +function collectDump (handle, timeout = 1000 * 60 * 2) { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + reject(Error('Timeout while waiting for dump')) + }, timeout) + let total = '' + function process (log) { + const data = log.toString() + total += data + for (const line of total.split('\n')) { + if (line.includes('') && line.includes('')) { + const blockData = line.split('')[1].split('')[0] + clearTimeout(timer) + resolve(blockData) + handle.stdout.off('data', process) + return + } + } + } + handle.stdout.on('data', process) + }) +} + +function injectPack (/** @type {import('minecraft-bedrock-server').BedrockVanillaServer} */ server, scriptVersion) { + server.clearBehaviorPacks() + server.addQuickScript({ + manifest: { + format_version: 2, + header: { + allow_random_seed: false, + description: 'DataExtractor', + name: 'DataExtractor', + platform_locked: false, + uuid: 'f604a121-974a-3e04-927a-8a1c9518c96a', + version: [1, 0, 0], + min_engine_version: [1, 20, 0] + }, + modules: [{ + type: 'script', + language: 'javascript', + uuid: 'fa04a121-974a-3e04-927a-8a1c9518c96a', + entry: 'scripts/main.js', + version: [0, 1, 0] + }], + dependencies: [ + { module_name: '@minecraft/server', version: scriptVersion || '1.0.0-beta' } + ] + }, + scripts: { + 'scripts/main.js': path.join(__dirname, 'serverScript.js') + } + }, true, true) + server.toggleExperiments({ gametest: true }) +} diff --git a/.github/helper-bot/update3.js b/.github/helper-bot/update3.js new file mode 100644 index 00000000..0a42d4b9 --- /dev/null +++ b/.github/helper-bot/update3.js @@ -0,0 +1,40 @@ +const fs = require('fs') +const github = require('gh-helpers')() + +async function main () { + const stages = ['stage1.txt', 'stage2.txt', 'stage4.txt'] + const allStages = fs.createWriteStream('merged.txt') + for (const stage of stages) { + allStages.write(fs.readFileSync(stage, 'latin1')) + } + allStages.end(upload) +} + +async function upload () { + const artifact = await github.artifacts.createTextArtifact('updateData-' + process.env.UPDATE_VERSION, { + extracted: fs.readFileSync('merged.txt', 'latin1'), + collected: JSON.stringify(require('./collected.json')), + collectedBlockData: JSON.stringify(require('./collectedBlockData.json')) + }) + console.log('Created artifact', artifact) + const dispatch = await github.sendWorkflowDispatch({ + repo: 'llm-services', + workflow: 'dispatch.yml', + branch: 'main', + inputs: { + action: 'minecraft/bedrockDataUpdate', + payload: JSON.stringify({ + repo: await github.getRepoDetails(), + artifactId: artifact.id, + artifactSize: artifact.size, + updateVersion: process.env.UPDATE_VERSION, + serverVersion: process.env.SERVER_VERSION, + protocolVersion: process.env.PROTOCOL_VERSION, + issueNo: process.env.ISSUE_NUMBER + }) + } + }) + console.log('Sent workflow dispatch', dispatch) +} + +main() diff --git a/.github/workflows/update-helper.yml b/.github/workflows/update-helper.yml index 168dd044..8c2f6c01 100644 --- a/.github/workflows/update-helper.yml +++ b/.github/workflows/update-helper.yml @@ -8,17 +8,89 @@ jobs: helper: name: update-checker runs-on: ubuntu-latest + outputs: + updateVersion: ${{ steps.helperRun.outputs.updateVersion }} + issueNumber: ${{ steps.helperRun.outputs.issueNumber }} steps: - name: Checkout repository uses: actions/checkout@master - name: Set up Node.js uses: actions/setup-node@master with: - node-version: 18.0.0 - - name: Install Github Actions helper - run: npm i gh-helpers + node-version: 20.0.0 + - name: Install deps in helper-bot + run: npm install + working-directory: .github/helper-bot # The env vars contain the relevant trigger information, so we don't need to pass it - name: Runs helper - run: cd .github/helper-bot && node index.js + id: helperRun + run: node index.js + working-directory: .github/helper-bot env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.PAT_PASSWORD }} + + update: + name: Update + needs: helper + runs-on: ubuntu-latest + # run if updateVersion exists + if: ${{ needs.helper.outputs.updateVersion }} + steps: + - name: Checkout repository + uses: actions/checkout@master + - name: Set up Node.js + uses: actions/setup-node@master + with: + node-version: 20.0.0 + - name: Compile disa + run: clang++ -std=c++20 -O3 -o disa.exe disa.cpp + working-directory: .github/helper-bot + - name: Compile pdba + run: clang++-14 -g `llvm-config-14 --cxxflags --ldflags --libs` -o pdba.exe pdba.cpp + working-directory: .github/helper-bot + - name: Install bedrock-protocol + run: npm install gh-helpers + # This step sets the "serverVersion" and "serverPath" outputs + # Note, sometimes the serverVersionStr != clientVersionStr, so we handle that too via protocol check + - name: Update Step 1 + id: update1Run + run: cd .github/helper-bot && node update1.js + # Pass the output from the helper job to the update job + env: + GITHUB_TOKEN: ${{ secrets.PAT_PASSWORD }} + UPDATE_VERSION: ${{ needs.helper.outputs.updateVersion }} + ISSUE_NUMBER: ${{ needs.helper.outputs.issueNumber }} + # The above will return a "needsUpdate" output. + - name: Dump .rodata + if: steps.update1Run.outputs.needsUpdate + run: | + objcopy -O binary -j .rodata ${{ steps.update1Run.outputs.serverBin }} .github/helper-bot/rodata.bin + readelf -S ${{ steps.update1Run.outputs.serverBin }} | grep .rodata >> .github/helper-bot/rodata.bin + - name: Process Linux server (Stage 1) + if: steps.update1Run.outputs.needsUpdate + run: objdump -d --demangle --no-show-raw-insn ${{ steps.update1Run.outputs.serverBin }} | .github/helper-bot/disa.exe -s1 .github/helper-bot/rodata.bin > .github/helper-bot/stage1.txt + - name: Process Windows bin (Stage 2) + if: steps.update1Run.outputs.needsUpdate + run: objdump -d --demangle -M intel ${{ steps.update1Run.outputs.serverWinBin }} | .github/helper-bot/disa.exe -s2 .github/helper-bot/stage1.txt > .github/helper-bot/stage2.txt + - name: Resolve PDB symbols (Stage 4) + if: steps.update1Run.outputs.needsUpdate + run: | + WIN_BIN_TEXT_OFF=`objdump -h ${{ steps.update1Run.outputs.serverWinBin }} | grep ".text" | cut -b 29-45` + WIN_BIN_RELOC_OFF=`objdump -h ${{ steps.update1Run.outputs.serverWinBin }} | grep ".reloc" | cut -b 29-45` + llvm-pdbutil-14 dump --all ${{ steps.update1Run.outputs.serverWinPdb }} | .github/helper-bot/pdba.exe $WIN_BIN_TEXT_OFF $WIN_BIN_RELOC_OFF > .github/helper-bot/stage4.txt + # We use Artifacts API for cross repo com which is only avaliable within an Action, but not normal run jobs. + # So we need to store the keys in the env for update3 to use to do a workflow dispatch + # as in https://github.com/extremeheat/gh-helpers/blob/main/.github/workflows/ci.yml + - name: Expose GitHub Runtime + if: steps.update1Run.outputs.needsUpdate + uses: crazy-max/ghaction-github-runtime@v3 + - name: Aggregate and finish + if: steps.update1Run.outputs.needsUpdate + run: node update3.js + working-directory: .github/helper-bot + env: + GITHUB_TOKEN: ${{ secrets.PAT_PASSWORD }} + UPDATE_VERSION: ${{ needs.helper.outputs.updateVersion }} + SERVER_VERSION: ${{ steps.update1Run.outputs.serverVersion }} + PROTOCOL_VERSION: ${{ steps.update1Run.outputs.protocolVersion }} + ISSUE_NUMBER: ${{ needs.helper.outputs.issueNumber }} \ No newline at end of file diff --git a/.gitpod.DockerFile b/.gitpod.DockerFile deleted file mode 100644 index 061bf596..00000000 --- a/.gitpod.DockerFile +++ /dev/null @@ -1,8 +0,0 @@ -FROM gitpod/workspace-full:latest - -RUN bash -c ". .nvm/nvm.sh \ - && nvm install 14 \ - && nvm use 14 \ - && nvm alias default 14" - -RUN echo "nvm use default &>/dev/null" >> ~/.bashrc.d/51-nvm-fix \ No newline at end of file diff --git a/index.d.ts b/index.d.ts index c93b5a99..84a47a32 100644 --- a/index.d.ts +++ b/index.d.ts @@ -3,11 +3,12 @@ import { Realm } from 'prismarine-realms' import { ServerDeviceCodeResponse } from 'prismarine-auth' declare module 'bedrock-protocol' { + // Note: this tracks minecraft for iOS's version numbers, not server versions. type Version = '1.20.40' | '1.20.30' | '1.20.10' | '1.20.0' | '1.19.80' | '1.19.70' | '1.19.63' | '1.19.62' | '1.19.60' | '1.19.51' | '1.19.50' | '1.19.41' | '1.19.40' | '1.19.31' | '1.19.30' | '1.19.22' | '1.19.21' | '1.19.20' | '1.19.11' | '1.19.10' | '1.19.2' | '1.19.1' | '1.18.31' | '1.18.30' | '1.18.12' | '1.18.11' | '1.18.10' | '1.18.2' | '1.18.1' | '1.18.0' | '1.17.41' | '1.17.40' | '1.17.34' | '1.17.30' | '1.17.11' | '1.17.10' | '1.17.0' | '1.16.220' | '1.16.210' | '1.16.201' export interface Options { // The string version to start the client or server as - version?: Version + version?: string // For the client, the host of the server to connect to (default: 127.0.0.1) // For the server, the host to bind to (default: 0.0.0.0) host: string diff --git a/package.json b/package.json index 18fe1e46..9feafe4a 100644 --- a/package.json +++ b/package.json @@ -6,10 +6,11 @@ "types": "index.d.ts", "scripts": { "build": "cd tools && node compileProtocol.js", - "test": "mocha --bail --exit", + "mocha": "mocha --bail --exit", "pretest": "npm run lint", + "test": "mocha --bail --exit", "lint": "standard", - "vanillaServer": "node tools/startVanillaServer.js", + "vanillaServer": "minecraft-bedrock-server --root tools", "dumpPackets": "node tools/genPacketDumps.js", "fix": "standard --fix" }, @@ -40,6 +41,7 @@ "bedrock-protocol": "file:.", "bedrock-provider": "^2.0.0", "leveldb-zlib": "^1.0.1", + "minecraft-bedrock-server": "^1.2.0", "mocha": "^10.0.0", "protodef-yaml": "^1.1.0", "standard": "^17.0.0-2" diff --git a/src/createClient.js b/src/createClient.js index 154a8ae7..4318889b 100644 --- a/src/createClient.js +++ b/src/createClient.js @@ -83,10 +83,10 @@ function connect (client) { }) } -async function ping ({ host, port }) { +async function ping ({ host, port, timeout }) { const con = new RakClient({ host, port }) try { - return advertisement.fromServerName(await con.ping()) + return advertisement.fromServerName(await con.ping(timeout)) } finally { con.close() } diff --git a/src/datatypes/util.js b/src/datatypes/util.js index 7070ce50..5c1c9261 100644 --- a/src/datatypes/util.js +++ b/src/datatypes/util.js @@ -22,10 +22,10 @@ function sleep (ms) { async function waitFor (cb, withTimeout, onTimeout) { let t - const ret = await Promise.race([ - new Promise((resolve, reject) => cb(resolve, reject)), - new Promise(resolve => { t = setTimeout(() => resolve('timeout'), withTimeout) }) - ]) + const ret = await new Promise((resolve) => { + t = setTimeout(() => resolve('timeout'), withTimeout) + cb(resolve) + }) clearTimeout(t) if (ret === 'timeout') await onTimeout() return ret diff --git a/src/options.js b/src/options.js index ed4ff6cc..795f5957 100644 --- a/src/options.js +++ b/src/options.js @@ -2,7 +2,7 @@ const mcData = require('minecraft-data') // Minimum supported version (< will be kicked) const MIN_VERSION = '1.16.201' -// Currently supported verson. Note, clients with newer versions can still connect as long as data is in minecraft-data +// Currently supported version. Note, clients with newer versions can still connect as long as data is in minecraft-data const CURRENT_VERSION = '1.20.71' const Versions = Object.fromEntries(mcData.versions.bedrock.filter(e => e.releaseType === 'release').map(e => [e.minecraftVersion, e.version])) diff --git a/test/internal.js b/test/internal.js index 6b37d957..cee27218 100644 --- a/test/internal.js +++ b/test/internal.js @@ -4,7 +4,7 @@ const { ping } = require('../src/createClient') const { CURRENT_VERSION } = require('../src/options') const { join } = require('path') const { waitFor } = require('../src/datatypes/util') -const { getPort } = require('./util') +const { getPort } = require('./util/util') // First we need to dump some packets that a vanilla server would send a vanilla // client. Then we can replay those back in our custom server. @@ -14,7 +14,7 @@ function prepare (version) { async function startTest (version = CURRENT_VERSION, ok) { await prepare(version) - const Item = require('../types/Item')(version) + const Item = require('./util/Item')(version) const port = await getPort() const server = new Server({ host: '0.0.0.0', port, version, offline: true }) @@ -202,7 +202,7 @@ async function requestChunks (version, x, z, radius) { return chunks } -async function timedTest (version, timeout = 1000 * 220) { +async function timedTest (version, timeout = 1000 * 60 * 6) { await waitFor((resolve, reject) => { // mocha eats up stack traces... startTest(version, resolve).catch(reject) @@ -212,5 +212,5 @@ async function timedTest (version, timeout = 1000 * 220) { console.info('✔ ok') } -// if (!module.parent) timedTest('1.19.10') +// if (!module.parent) timedTest('1.20.61') module.exports = { startTest, timedTest, requestChunks } diff --git a/test/internal.test.js b/test/internal.test.js index 1b6250ce..4309acc6 100644 --- a/test/internal.test.js +++ b/test/internal.test.js @@ -1,12 +1,11 @@ /* eslint-env jest */ - const { timedTest } = require('./internal') const { testedVersions } = require('../src/options') const { sleep } = require('../src/datatypes/util') describe('internal client/server test', function () { const vcount = testedVersions.length - this.timeout(vcount * 80 * 1000) + this.timeout(vcount * 7 * 60 * 1000) // upto 7 minutes per version for (const version of testedVersions) { it('connects ' + version, async () => { diff --git a/test/proxy.js b/test/proxy.js index 345e89fc..09506c76 100644 --- a/test/proxy.js +++ b/test/proxy.js @@ -1,6 +1,6 @@ const { createClient, Server, Relay } = require('bedrock-protocol') const { sleep, waitFor } = require('../src/datatypes/util') -const { getPort } = require('./util') +const { getPort } = require('./util/util') function proxyTest (version, raknetBackend = 'raknet-native', timeout = 1000 * 40) { console.log('with raknet backend', raknetBackend) diff --git a/test/proxy.test.js b/test/proxy.test.js index 70ea5f19..d4fc8acc 100644 --- a/test/proxy.test.js +++ b/test/proxy.test.js @@ -5,7 +5,7 @@ const { sleep } = require('../src/datatypes/util') describe('proxies client/server', function () { const vcount = testedVersions.length - this.timeout(vcount * 30 * 1000) + this.timeout(vcount * 7 * 60 * 1000) // upto 7 minutes per version for (const version of testedVersions) { it('proxies ' + version, async () => { diff --git a/types/Item.js b/test/util/Item.js similarity index 97% rename from types/Item.js rename to test/util/Item.js index 5de23d29..239dcad2 100644 --- a/types/Item.js +++ b/test/util/Item.js @@ -1,4 +1,4 @@ -const { Versions } = require('../src/options') +const { Versions } = require('../../src/options') module.exports = (version) => class Item { diff --git a/test/util.js b/test/util/util.js similarity index 96% rename from test/util.js rename to test/util/util.js index 2876a67f..92899931 100644 --- a/test/util.js +++ b/test/util/util.js @@ -1,12 +1,12 @@ -const net = require('net') - -const getPort = () => new Promise(resolve => { - const server = net.createServer() - server.listen(0, '127.0.0.1') - server.on('listening', () => { - const { port } = server.address() - server.close(() => resolve(port)) - }) -}) - -module.exports = { getPort } +const net = require('net') + +const getPort = () => new Promise(resolve => { + const server = net.createServer() + server.listen(0, '127.0.0.1') + server.on('listening', () => { + const { port } = server.address() + server.close(() => resolve(port)) + }) +}) + +module.exports = { getPort } diff --git a/test/vanilla.js b/test/vanilla.js index 7a24008c..5323cd95 100644 --- a/test/vanilla.js +++ b/test/vanilla.js @@ -1,12 +1,10 @@ -// process.env.DEBUG = 'minecraft-protocol raknet' -const vanillaServer = require('../tools/startVanillaServer') +process.env.DEBUG = 'minecraft-protocol raknet' +const vanillaServer = require('minecraft-bedrock-server') const { Client } = require('../src/client') const { waitFor } = require('../src/datatypes/util') -const { getPort } = require('./util') +const { getPort } = require('./util/util') async function test (version) { - const ChunkColumn = require('bedrock-provider').chunk('bedrock_' + (version.includes('1.19') ? '1.18.30' : version)) // TODO: Fix prismarine-chunk - // Start the server, wait for it to accept clients, throws on timeout const port = await getPort() const handle = await vanillaServer.startServerAndWait2(version, 1000 * 220, { 'server-port': port }) @@ -48,11 +46,6 @@ async function test (version) { client.queue('tick_sync', { request_time: BigInt(Date.now()), response_time: BigInt(Date.now()) }) }, 200) - client.on('level_chunk', async packet => { // Chunk read test - const cc = new ChunkColumn(packet.x, packet.z) - await cc.networkDecodeNoCache(packet.payload, packet.sub_chunk_count) - }) - console.log('Awaiting join') client.on('spawn', () => { diff --git a/test/vanilla.test.js b/test/vanilla.test.js index 38d1a7c4..7bf9fbcb 100644 --- a/test/vanilla.test.js +++ b/test/vanilla.test.js @@ -6,7 +6,7 @@ const { sleep } = require('../src/datatypes/util') describe('vanilla server test', function () { const vcount = testedVersions.length - this.timeout(vcount * 80 * 1000) + this.timeout(vcount * 7 * 60 * 1000) // upto 7 minutes per version for (const version of testedVersions) { it('client spawns ' + version, async () => { diff --git a/tools/dumpPackets.js b/tools/dumpPackets.js index 47cc30bb..0d33f2e6 100644 --- a/tools/dumpPackets.js +++ b/tools/dumpPackets.js @@ -2,7 +2,7 @@ // uses the same format as prismarine-packet-dumper const assert = require('assert') const fs = require('fs') -const vanillaServer = require('../tools/startVanillaServer') +const vanillaServer = require('minecraft-bedrock-server') const { Client } = require('../src/client') const { serialize, waitFor } = require('../src/datatypes/util') const { CURRENT_VERSION } = require('../src/options') @@ -16,7 +16,10 @@ async function dump (version) { const random = ((Math.random() * 100) | 0) const port = 19130 + random - const handle = await vanillaServer.startServerAndWait(version || CURRENT_VERSION, 1000 * 120, { 'server-port': port }) + const handle = await vanillaServer.startServerAndWait(version || CURRENT_VERSION, 1000 * 120, { + 'server-port': port, + root: __dirname + }) console.log('Started server') const client = new Client({ diff --git a/tools/genPacketDumps.js b/tools/genPacketDumps.js index 2e6420ff..5415fa67 100644 --- a/tools/genPacketDumps.js +++ b/tools/genPacketDumps.js @@ -1,12 +1,12 @@ // Collect sample packets needed for `serverTest.js` // process.env.DEBUG = 'minecraft-protocol' const fs = require('fs') -const vanillaServer = require('../tools/startVanillaServer') +const vanillaServer = require('minecraft-bedrock-server') const { Client } = require('../src/client') const { serialize, waitFor, getFiles } = require('../src/datatypes/util') const { CURRENT_VERSION } = require('../src/options') const { join } = require('path') -const { getPort } = require('../test/util') +const { getPort } = require('../test/util/util') function hasDumps (version) { const root = join(__dirname, `../data/${version}/sample/packets/`) @@ -25,7 +25,11 @@ async function dump (version, force = true) { const [port, v6] = [await getPort(), await getPort()] console.log('Starting dump server', version) - const handle = await vanillaServer.startServerAndWait2(version || CURRENT_VERSION, 1000 * 120, { 'server-port': port, 'server-portv6': v6 }) + const handle = await vanillaServer.startServerAndWait2(version || CURRENT_VERSION, 1000 * 60 * 3, { + 'server-port': port, + 'server-portv6': v6, + root: __dirname + }) console.log('Started dump server', version) const client = new Client({ diff --git a/tools/startVanillaServer.js b/tools/startVanillaServer.js deleted file mode 100644 index 6d77006c..00000000 --- a/tools/startVanillaServer.js +++ /dev/null @@ -1,156 +0,0 @@ -const http = require('https') -const fs = require('fs') -const cp = require('child_process') -const debug = process.env.CI ? console.debug : require('debug')('minecraft-protocol') -const https = require('https') -const { getFiles, waitFor } = require('../src/datatypes/util') - -function head (url) { - return new Promise((resolve, reject) => { - const req = http.request(url, { method: 'HEAD', timeout: 1000 }, resolve) - req.on('error', reject) - req.on('timeout', () => { req.destroy(); debug('HEAD request timeout'); reject(new Error('timeout')) }) - req.end() - }) -} -function get (url, outPath) { - const file = fs.createWriteStream(outPath) - return new Promise((resolve, reject) => { - https.get(url, { timeout: 1000 * 20 }, response => { - if (response.statusCode !== 200) return reject(new Error('Server returned code ' + response.statusCode)) - response.pipe(file) - file.on('finish', () => { - file.close() - resolve() - }) - }) - }) -} - -// Get the latest versions -// TODO: once we support multi-versions -function fetchLatestStable () { - get('https://raw.githubusercontent.com/minecraft-linux/mcpelauncher-versiondb/master/versions.json', 'versions.json') - const versions = JSON.parse(fs.readFileSync('./versions.json')) - const latest = versions[0] - return latest.version_name -} - -// Download + extract vanilla server and enter the directory -async function download (os, version, path = 'bds-') { - debug('Downloading server', os, version, 'into', path) - process.chdir(__dirname) - const verStr = version.split('.').slice(0, 3).join('.') - const dir = path + version - - if (fs.existsSync(dir) && getFiles(dir).length) { - process.chdir(path + version) // Enter server folder - return verStr - } - try { fs.mkdirSync(dir) } catch { } - - process.chdir(path + version) // Enter server folder - const url = (os, version) => `https://minecraft.azureedge.net/bin-${os}/bedrock-server-${version}.zip` - - let found = false - - for (let i = 0; i < 8; i++) { // Check for the latest server build for version (major.minor.patch.BUILD) - const u = url(os, `${verStr}.${String(i).padStart(2, '0')}`) - debug('Opening', u, Date.now()) - let ret - try { ret = await head(u) } catch (e) { continue } - if (ret.statusCode === 200) { - found = u - debug('Found server', ret.statusCode) - break - } - } - if (!found) throw Error('did not find server bin for ' + os + ' ' + version) - console.info('🔻 Downloading', found) - await get(found, 'bds.zip') - console.info('⚡ Unzipping') - // Unzip server - if (process.platform === 'linux') cp.execSync('unzip -u bds.zip && chmod +777 ./bedrock_server') - else cp.execSync('tar -xf bds.zip') - return verStr -} - -const defaultOptions = { - 'level-generator': '2', - 'server-port': '19130', - 'online-mode': 'false' -} - -// Setup the server -function configure (options = {}) { - const opts = { ...defaultOptions, ...options } - let config = fs.readFileSync('./server.properties', 'utf-8') - config += '\nplayer-idle-timeout=1\nallow-cheats=true\ndefault-player-permission-level=operator' - for (const o in opts) config += `\n${o}=${opts[o]}` - fs.writeFileSync('./server.properties', config) -} - -function run (inheritStdout = true) { - const exe = process.platform === 'win32' ? 'bedrock_server.exe' : './bedrock_server' - return cp.spawn(exe, inheritStdout ? { stdio: 'inherit' } : {}) -} - -let lastHandle - -// Run the server -async function startServer (version, onStart, options = {}) { - const os = process.platform === 'win32' ? 'win' : process.platform - if (os !== 'win' && os !== 'linux') { - throw Error('unsupported os ' + os) - } - await download(os, version, options.path) - configure(options) - const handle = lastHandle = run(!onStart) - handle.on('error', (...a) => { - console.warn('*** THE MINECRAFT PROCESS CRASHED ***', a) - handle.kill('SIGKILL') - }) - if (onStart) { - let stdout = '' - handle.stdout.on('data', data => { - stdout += data - if (stdout.includes('Server started')) onStart() - }) - handle.stdout.pipe(process.stdout) - handle.stderr.pipe(process.stdout) - } - return handle -} - -// Start the server and wait for it to be ready, with a timeout -async function startServerAndWait (version, withTimeout, options) { - let handle - await waitFor(async res => { - handle = await startServer(version, res, options) - }, withTimeout, () => { - handle?.kill() - throw new Error(`Server did not start on time (${withTimeout}ms, now ${Date.now()})`) - }) - return handle -} - -async function startServerAndWait2 (version, withTimeout, options) { - try { - return await startServerAndWait(version, 1000 * 60, options) - } catch (e) { - console.log(e) - console.log('^ Tring once more to start server in 10 seconds...') - lastHandle?.kill() - await new Promise(resolve => setTimeout(resolve, 10000)) - process.chdir(__dirname) - fs.rmSync('bds-' + version, { recursive: true }) - return await startServerAndWait(version, withTimeout, options) - } -} - -if (!module.parent) { - // if (process.argv.length < 3) throw Error('Missing version argument') - startServer(process.argv[2] || '1.17.10', null, process.argv[3] ? { 'server-port': process.argv[3], 'online-mode': !!process.argv[4] } : undefined) -} - -module.exports = { fetchLatestStable, startServer, startServerAndWait, startServerAndWait2 }