From b52edce342b1f633d3d95c13cb87415b61088779 Mon Sep 17 00:00:00 2001 From: extremeheat Date: Fri, 29 Mar 2024 01:45:38 -0400 Subject: [PATCH 01/50] update test timeouts, move server to separate package --- package.json | 6 +- test/internal.js | 8 +- test/internal.test.js | 3 +- test/proxy.test.js | 2 +- {types => test/util}/Item.js | 2 +- test/{ => util}/util.js | 24 +++--- test/vanilla.js | 12 +-- test/vanilla.test.js | 2 +- tools/startVanillaServer.js | 156 ----------------------------------- 9 files changed, 27 insertions(+), 188 deletions(-) rename {types => test/util}/Item.js (97%) rename test/{ => util}/util.js (96%) delete mode 100644 tools/startVanillaServer.js diff --git a/package.json b/package.json index 18fe1e46..9feafe4a 100644 --- a/package.json +++ b/package.json @@ -6,10 +6,11 @@ "types": "index.d.ts", "scripts": { "build": "cd tools && node compileProtocol.js", - "test": "mocha --bail --exit", + "mocha": "mocha --bail --exit", "pretest": "npm run lint", + "test": "mocha --bail --exit", "lint": "standard", - "vanillaServer": "node tools/startVanillaServer.js", + "vanillaServer": "minecraft-bedrock-server --root tools", "dumpPackets": "node tools/genPacketDumps.js", "fix": "standard --fix" }, @@ -40,6 +41,7 @@ "bedrock-protocol": "file:.", "bedrock-provider": "^2.0.0", "leveldb-zlib": "^1.0.1", + "minecraft-bedrock-server": "^1.2.0", "mocha": "^10.0.0", "protodef-yaml": "^1.1.0", "standard": "^17.0.0-2" diff --git a/test/internal.js b/test/internal.js index 6b37d957..cee27218 100644 --- a/test/internal.js +++ b/test/internal.js @@ -4,7 +4,7 @@ const { ping } = require('../src/createClient') const { CURRENT_VERSION } = require('../src/options') const { join } = require('path') const { waitFor } = require('../src/datatypes/util') -const { getPort } = require('./util') +const { getPort } = require('./util/util') // First we need to dump some packets that a vanilla server would send a vanilla // client. Then we can replay those back in our custom server. @@ -14,7 +14,7 @@ function prepare (version) { async function startTest (version = CURRENT_VERSION, ok) { await prepare(version) - const Item = require('../types/Item')(version) + const Item = require('./util/Item')(version) const port = await getPort() const server = new Server({ host: '0.0.0.0', port, version, offline: true }) @@ -202,7 +202,7 @@ async function requestChunks (version, x, z, radius) { return chunks } -async function timedTest (version, timeout = 1000 * 220) { +async function timedTest (version, timeout = 1000 * 60 * 6) { await waitFor((resolve, reject) => { // mocha eats up stack traces... startTest(version, resolve).catch(reject) @@ -212,5 +212,5 @@ async function timedTest (version, timeout = 1000 * 220) { console.info('✔ ok') } -// if (!module.parent) timedTest('1.19.10') +// if (!module.parent) timedTest('1.20.61') module.exports = { startTest, timedTest, requestChunks } diff --git a/test/internal.test.js b/test/internal.test.js index 1b6250ce..4309acc6 100644 --- a/test/internal.test.js +++ b/test/internal.test.js @@ -1,12 +1,11 @@ /* eslint-env jest */ - const { timedTest } = require('./internal') const { testedVersions } = require('../src/options') const { sleep } = require('../src/datatypes/util') describe('internal client/server test', function () { const vcount = testedVersions.length - this.timeout(vcount * 80 * 1000) + this.timeout(vcount * 7 * 60 * 1000) // upto 7 minutes per version for (const version of testedVersions) { it('connects ' + version, async () => { diff --git a/test/proxy.test.js b/test/proxy.test.js index 70ea5f19..d4fc8acc 100644 --- a/test/proxy.test.js +++ b/test/proxy.test.js @@ -5,7 +5,7 @@ const { sleep } = require('../src/datatypes/util') describe('proxies client/server', function () { const vcount = testedVersions.length - this.timeout(vcount * 30 * 1000) + this.timeout(vcount * 7 * 60 * 1000) // upto 7 minutes per version for (const version of testedVersions) { it('proxies ' + version, async () => { diff --git a/types/Item.js b/test/util/Item.js similarity index 97% rename from types/Item.js rename to test/util/Item.js index 5de23d29..239dcad2 100644 --- a/types/Item.js +++ b/test/util/Item.js @@ -1,4 +1,4 @@ -const { Versions } = require('../src/options') +const { Versions } = require('../../src/options') module.exports = (version) => class Item { diff --git a/test/util.js b/test/util/util.js similarity index 96% rename from test/util.js rename to test/util/util.js index 2876a67f..92899931 100644 --- a/test/util.js +++ b/test/util/util.js @@ -1,12 +1,12 @@ -const net = require('net') - -const getPort = () => new Promise(resolve => { - const server = net.createServer() - server.listen(0, '127.0.0.1') - server.on('listening', () => { - const { port } = server.address() - server.close(() => resolve(port)) - }) -}) - -module.exports = { getPort } +const net = require('net') + +const getPort = () => new Promise(resolve => { + const server = net.createServer() + server.listen(0, '127.0.0.1') + server.on('listening', () => { + const { port } = server.address() + server.close(() => resolve(port)) + }) +}) + +module.exports = { getPort } diff --git a/test/vanilla.js b/test/vanilla.js index 7a24008c..55758d99 100644 --- a/test/vanilla.js +++ b/test/vanilla.js @@ -1,12 +1,10 @@ -// process.env.DEBUG = 'minecraft-protocol raknet' -const vanillaServer = require('../tools/startVanillaServer') +process.env.DEBUG = 'minecraft-protocol raknet' +const vanillaServer = require('minecraft-bedrock-server') const { Client } = require('../src/client') const { waitFor } = require('../src/datatypes/util') -const { getPort } = require('./util') +const { getPort } = require('./util/util') async function test (version) { - const ChunkColumn = require('bedrock-provider').chunk('bedrock_' + (version.includes('1.19') ? '1.18.30' : version)) // TODO: Fix prismarine-chunk - // Start the server, wait for it to accept clients, throws on timeout const port = await getPort() const handle = await vanillaServer.startServerAndWait2(version, 1000 * 220, { 'server-port': port }) @@ -48,10 +46,6 @@ async function test (version) { client.queue('tick_sync', { request_time: BigInt(Date.now()), response_time: BigInt(Date.now()) }) }, 200) - client.on('level_chunk', async packet => { // Chunk read test - const cc = new ChunkColumn(packet.x, packet.z) - await cc.networkDecodeNoCache(packet.payload, packet.sub_chunk_count) - }) console.log('Awaiting join') diff --git a/test/vanilla.test.js b/test/vanilla.test.js index 38d1a7c4..7bf9fbcb 100644 --- a/test/vanilla.test.js +++ b/test/vanilla.test.js @@ -6,7 +6,7 @@ const { sleep } = require('../src/datatypes/util') describe('vanilla server test', function () { const vcount = testedVersions.length - this.timeout(vcount * 80 * 1000) + this.timeout(vcount * 7 * 60 * 1000) // upto 7 minutes per version for (const version of testedVersions) { it('client spawns ' + version, async () => { diff --git a/tools/startVanillaServer.js b/tools/startVanillaServer.js deleted file mode 100644 index 6d77006c..00000000 --- a/tools/startVanillaServer.js +++ /dev/null @@ -1,156 +0,0 @@ -const http = require('https') -const fs = require('fs') -const cp = require('child_process') -const debug = process.env.CI ? console.debug : require('debug')('minecraft-protocol') -const https = require('https') -const { getFiles, waitFor } = require('../src/datatypes/util') - -function head (url) { - return new Promise((resolve, reject) => { - const req = http.request(url, { method: 'HEAD', timeout: 1000 }, resolve) - req.on('error', reject) - req.on('timeout', () => { req.destroy(); debug('HEAD request timeout'); reject(new Error('timeout')) }) - req.end() - }) -} -function get (url, outPath) { - const file = fs.createWriteStream(outPath) - return new Promise((resolve, reject) => { - https.get(url, { timeout: 1000 * 20 }, response => { - if (response.statusCode !== 200) return reject(new Error('Server returned code ' + response.statusCode)) - response.pipe(file) - file.on('finish', () => { - file.close() - resolve() - }) - }) - }) -} - -// Get the latest versions -// TODO: once we support multi-versions -function fetchLatestStable () { - get('https://raw.githubusercontent.com/minecraft-linux/mcpelauncher-versiondb/master/versions.json', 'versions.json') - const versions = JSON.parse(fs.readFileSync('./versions.json')) - const latest = versions[0] - return latest.version_name -} - -// Download + extract vanilla server and enter the directory -async function download (os, version, path = 'bds-') { - debug('Downloading server', os, version, 'into', path) - process.chdir(__dirname) - const verStr = version.split('.').slice(0, 3).join('.') - const dir = path + version - - if (fs.existsSync(dir) && getFiles(dir).length) { - process.chdir(path + version) // Enter server folder - return verStr - } - try { fs.mkdirSync(dir) } catch { } - - process.chdir(path + version) // Enter server folder - const url = (os, version) => `https://minecraft.azureedge.net/bin-${os}/bedrock-server-${version}.zip` - - let found = false - - for (let i = 0; i < 8; i++) { // Check for the latest server build for version (major.minor.patch.BUILD) - const u = url(os, `${verStr}.${String(i).padStart(2, '0')}`) - debug('Opening', u, Date.now()) - let ret - try { ret = await head(u) } catch (e) { continue } - if (ret.statusCode === 200) { - found = u - debug('Found server', ret.statusCode) - break - } - } - if (!found) throw Error('did not find server bin for ' + os + ' ' + version) - console.info('🔻 Downloading', found) - await get(found, 'bds.zip') - console.info('⚡ Unzipping') - // Unzip server - if (process.platform === 'linux') cp.execSync('unzip -u bds.zip && chmod +777 ./bedrock_server') - else cp.execSync('tar -xf bds.zip') - return verStr -} - -const defaultOptions = { - 'level-generator': '2', - 'server-port': '19130', - 'online-mode': 'false' -} - -// Setup the server -function configure (options = {}) { - const opts = { ...defaultOptions, ...options } - let config = fs.readFileSync('./server.properties', 'utf-8') - config += '\nplayer-idle-timeout=1\nallow-cheats=true\ndefault-player-permission-level=operator' - for (const o in opts) config += `\n${o}=${opts[o]}` - fs.writeFileSync('./server.properties', config) -} - -function run (inheritStdout = true) { - const exe = process.platform === 'win32' ? 'bedrock_server.exe' : './bedrock_server' - return cp.spawn(exe, inheritStdout ? { stdio: 'inherit' } : {}) -} - -let lastHandle - -// Run the server -async function startServer (version, onStart, options = {}) { - const os = process.platform === 'win32' ? 'win' : process.platform - if (os !== 'win' && os !== 'linux') { - throw Error('unsupported os ' + os) - } - await download(os, version, options.path) - configure(options) - const handle = lastHandle = run(!onStart) - handle.on('error', (...a) => { - console.warn('*** THE MINECRAFT PROCESS CRASHED ***', a) - handle.kill('SIGKILL') - }) - if (onStart) { - let stdout = '' - handle.stdout.on('data', data => { - stdout += data - if (stdout.includes('Server started')) onStart() - }) - handle.stdout.pipe(process.stdout) - handle.stderr.pipe(process.stdout) - } - return handle -} - -// Start the server and wait for it to be ready, with a timeout -async function startServerAndWait (version, withTimeout, options) { - let handle - await waitFor(async res => { - handle = await startServer(version, res, options) - }, withTimeout, () => { - handle?.kill() - throw new Error(`Server did not start on time (${withTimeout}ms, now ${Date.now()})`) - }) - return handle -} - -async function startServerAndWait2 (version, withTimeout, options) { - try { - return await startServerAndWait(version, 1000 * 60, options) - } catch (e) { - console.log(e) - console.log('^ Tring once more to start server in 10 seconds...') - lastHandle?.kill() - await new Promise(resolve => setTimeout(resolve, 10000)) - process.chdir(__dirname) - fs.rmSync('bds-' + version, { recursive: true }) - return await startServerAndWait(version, withTimeout, options) - } -} - -if (!module.parent) { - // if (process.argv.length < 3) throw Error('Missing version argument') - startServer(process.argv[2] || '1.17.10', null, process.argv[3] ? { 'server-port': process.argv[3], 'online-mode': !!process.argv[4] } : undefined) -} - -module.exports = { fetchLatestStable, startServer, startServerAndWait, startServerAndWait2 } From 8f143ef45da988d789cfa688992ec4d54e6752db Mon Sep 17 00:00:00 2001 From: extremeheat Date: Fri, 29 Mar 2024 01:50:13 -0400 Subject: [PATCH 02/50] update helper-bot workflows --- .github/helper-bot/index.js | 7 +++++-- .github/helper-bot/package.json | 5 +++++ .github/workflows/update-helper.yml | 8 +++++--- 3 files changed, 15 insertions(+), 5 deletions(-) create mode 100644 .github/helper-bot/package.json diff --git a/.github/helper-bot/index.js b/.github/helper-bot/index.js index 8dffc6c0..5b76b744 100644 --- a/.github/helper-bot/index.js +++ b/.github/helper-bot/index.js @@ -1,6 +1,7 @@ // Automatic version update checker for Minecraft bedrock edition. const fs = require('fs') const cp = require('child_process') +const core = require('@actions/core') const helper = require('gh-helpers')() const latestVesionEndpoint = 'https://itunes.apple.com/lookup?bundleId=com.mojang.minecraftpe&time=' + Date.now() const changelogURL = 'https://feedback.minecraft.net/hc/en-us/sections/360001186971-Release-Changelogs' @@ -127,9 +128,11 @@ async function fetchLatest () { }) if (issueStatus.isOpen) { - helper.updateIssue(issueStatus.id, issuePayload) + await helper.updateIssue(issueStatus.id, issuePayload) } else { - helper.createIssue(issuePayload) + const issue = await helper.createIssue(issuePayload) + core.setOutput('updateVersion', version) + core.setOutput('issueNumber', issue.number) } fs.writeFileSync('./issue.md', issuePayload.body) diff --git a/.github/helper-bot/package.json b/.github/helper-bot/package.json new file mode 100644 index 00000000..658d5db8 --- /dev/null +++ b/.github/helper-bot/package.json @@ -0,0 +1,5 @@ +{ + "dependencies": { + "gh-helpers": "^0.2.0" + } +} diff --git a/.github/workflows/update-helper.yml b/.github/workflows/update-helper.yml index 168dd044..551746d8 100644 --- a/.github/workflows/update-helper.yml +++ b/.github/workflows/update-helper.yml @@ -15,10 +15,12 @@ jobs: uses: actions/setup-node@master with: node-version: 18.0.0 - - name: Install Github Actions helper - run: npm i gh-helpers + - name: Install deps in helper-bot + run: npm install + working-directory: .github/helper-bot # The env vars contain the relevant trigger information, so we don't need to pass it - name: Runs helper - run: cd .github/helper-bot && node index.js + run: node index.js + working-directory: .github/helper-bot env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 28cb0a82c1b61db2db8f0c2e0766ac6b30d44a70 Mon Sep 17 00:00:00 2001 From: extremeheat Date: Sun, 31 Mar 2024 08:09:12 -0400 Subject: [PATCH 03/50] start work on update workflow --- .github/helper-bot/.clang-format | 2 + .github/helper-bot/disa.cpp | 460 +++++++++++++++++++++++++++++++ .github/helper-bot/download.js | 28 ++ .github/helper-bot/index.js | 8 +- .github/helper-bot/pdba.cpp | 42 +++ test/vanilla.js | 1 - 6 files changed, 535 insertions(+), 6 deletions(-) create mode 100644 .github/helper-bot/.clang-format create mode 100644 .github/helper-bot/disa.cpp create mode 100644 .github/helper-bot/download.js create mode 100644 .github/helper-bot/pdba.cpp diff --git a/.github/helper-bot/.clang-format b/.github/helper-bot/.clang-format new file mode 100644 index 00000000..ab45593a --- /dev/null +++ b/.github/helper-bot/.clang-format @@ -0,0 +1,2 @@ +ColumnLimit: 120 +SortIncludes: false \ No newline at end of file diff --git a/.github/helper-bot/disa.cpp b/.github/helper-bot/disa.cpp new file mode 100644 index 00000000..970cb1b0 --- /dev/null +++ b/.github/helper-bot/disa.cpp @@ -0,0 +1,460 @@ +#include +#include +#include +#include +#include +#include +#include + +void ZeroMemory(char *buffer, int size) { + for (int i = 0; i < size; i++) { + buffer[i] = 0; + } +} + +unsigned int hexStr2Int(const std::string_view &hexStr) { + const char *hexStrC = hexStr.data(); + unsigned int value = 0; + for (size_t i = 0; i < hexStr.size(); i++) { + char c = hexStrC[i]; + if (c >= '0' && c <= '9') { + value = (value << 4) + (c - '0'); + } else if (c >= 'a' && c <= 'f') { + value = (value << 4) + (c - 'a' + 10); + } else if (c >= 'A' && c <= 'F') { + value = (value << 4) + (c - 'A' + 10); + } + } + return value; +} + +unsigned int hexStr2IntLE(const std::string_view &hexStr) { + unsigned int value = hexStr2Int(hexStr); + unsigned int swapped = ((value >> 24) & 0xff) | // move byte 3 to byte 0 + ((value << 8) & 0xff0000) | // move byte 1 to byte 2 + ((value >> 8) & 0xff00) | // move byte 2 to byte 1 + ((value << 24) & 0xff000000); // byte 0 to byte 3 + return swapped; +} + +std::map stringsMap; +void loadStrings(std::string filePath) { + // Load the strings file TSV into a map + std::ifstream stringsStream(filePath, std::ios::binary); + if (!stringsStream.is_open()) { + std::cerr << "Failed to open file: " << filePath << std::endl; + return; + } + const int bufferSize = 1024 * 64; + char buffer[bufferSize]; + while (stringsStream.getline(buffer, bufferSize)) { + std::string_view line(buffer); + size_t pos = line.find('\t'); + if (pos != std::string::npos) { + std::string_view key = line.substr(0, pos); + std::string_view value = line.substr(pos + 1, line.size() - pos - 1); + unsigned int keyInt = hexStr2Int(key); + stringsMap[keyInt] = std::string(value); + } + } + stringsStream.close(); +} + +struct CurrentBlockData { + std::string blockName; + std::string blockClass; + unsigned int breakTimeAddr; + std::vector stateKeys; +}; +std::vector blockData; + +bool hasSeenBlockClass(const std::string_view &blockClass) { + for (auto &block : blockData) { + if (block.blockClass == blockClass) { + return true; + } + } + return false; +} + +void loadDisassembly(std::string filePath) { + // *uses AT&T syntax ; too late to change now + std::istream *disStream = &std::cin; + if (!filePath.empty()) { + disStream = new std::ifstream(filePath, std::ios::binary); + if (!((std::ifstream *)disStream)->is_open()) { + std::cerr << "Failed to open file: " << filePath << std::endl; + return; + } + } + + // 64KB buffer + const int bufferSize = 1024 * 64; + char buffer[bufferSize]; + std::string trackingBlock; + std::string trackingState; + bool isInGlobalBlock = false; + bool isInBlockRegistry = false; + + std::string inStateSerializer; + std::map> stateEntries; + + unsigned int lastLastLoadedAddress; + unsigned int lastLoadedAddress; + std::string lastLoadedAddressAbsMovStr; + + std::optional currentBlockData; + + while (disStream->getline(buffer, bufferSize)) { + // movabs $0x116b2c0, %rbx -> move the address to rbx + if (buffer[10] == 'm' && buffer[11] == 'o' && buffer[12] == 'v' && buffer[13] == 'a' && buffer[14] == 'b' && + buffer[15] == 's') { + std::string_view line(buffer); + size_t pos = line.find("$"); + size_t endPos = line.find(","); + if (pos != std::string::npos && endPos != std::string::npos) { + auto addressStr = line.substr(pos + 1, endPos - pos - 1); + lastLoadedAddressAbsMovStr = addressStr; + // auto addressInt = hexStr2Int(addressStr); + + // if we are tracking a block, then print the constant + if (!trackingBlock.empty()) { + // if line includes '#', then split by the comment and get the comment + if (stringsMap.find(lastLoadedAddress) != stringsMap.end()) { + std::cout << "BlockID\t" << trackingBlock << "\t" << addressStr << "\t" << stringsMap[lastLoadedAddress] + << std::endl; + } + } + } + } + + // callq 745ef90 (HashedString const&, int&&)> + if (buffer[10] == 'c' && buffer[11] == 'a' && buffer[12] == 'l' && buffer[13] == 'l') { + std::string_view line(buffer); + + // if line contains registerBlock, then it's a block registration + size_t registerPos = line.find("registerBlock<"); + if (registerPos != std::string::npos) { + if (currentBlockData.has_value()) { + blockData.push_back(currentBlockData.value()); + currentBlockData.reset(); + } + + // class name is between "registerBlock<" and "," + size_t classStart = registerPos; + size_t classEnd = line.find(",", classStart); + auto blockClass = line.substr(classStart + 14, classEnd - classStart - 14); + if (trackingBlock.empty()) { + if (!hasSeenBlockClass(blockClass)) { + std::cerr << "? Unloaded Block registration: " << line << std::endl; + } + } else { + currentBlockData = CurrentBlockData{.blockClass = std::string(blockClass)}; + // std::cout << "BlockRegistration\t" << trackingBlock << "\t" << blockClass << std::endl; + currentBlockData->blockName = trackingBlock; + } + } + + if (currentBlockData.has_value()) { + // 74061b8: callq 6bd9f60 + size_t destroyPos = line.find("setDestroyTime"); + if (destroyPos != std::string::npos) { + if (line.find(", float") != std::string::npos) { + // the last last loaded address is the address of the destroy time value + currentBlockData->breakTimeAddr = lastLastLoadedAddress; + } else { + // the last loaded address is the address of the destroy time value + currentBlockData->breakTimeAddr = lastLoadedAddress; + } + } + } + } + + if (buffer[10] == 'l' && buffer[11] == 'e' && buffer[12] == 'a') { + std::string_view line(buffer); + + size_t pos = line.find("#"); + if (pos != std::string::npos) { + // std::cout << trackingBlock << " " << buffer << std::endl; + auto comment = line.substr(pos + 2, line.size() - pos - 1); + // above bounds to skip the comment+space + // split the comment by the first space + auto addressPos = comment.find(" "); + if (addressPos != std::string::npos) { + auto addressStr = comment.substr(0, addressPos); + auto addressInt = hexStr2Int(addressStr); + + lastLastLoadedAddress = lastLoadedAddress; + lastLoadedAddress = addressInt; + + if (inStateSerializer.size() > 0) { + // we are interested in capturing all loaded constants inside the state serializer + if (stringsMap.find(addressInt) != stringsMap.end()) { + stateEntries[inStateSerializer].push_back(stringsMap[addressInt]); + } + } + } + + size_t pos = line.find("VanillaBlockTypeIds::"); + if (pos != std::string::npos) { + trackingBlock = line.substr(pos + 21, line.size() - pos - 22); + } + + if (currentBlockData.has_value()) { + // 758eaa3: lea 0x1145dae(%rip),%rsi # 86d4858 + size_t statesPos = line.find("VanillaStates::"); + if (statesPos != std::string::npos) { + // ensure there's no + in the symbol + if (comment.find("+") != std::string::npos) { + continue; + } + auto end = comment.find(">"); + auto states = line.substr(statesPos + 15, line.size() - statesPos - 16); + auto statesStr = std::string(states); + currentBlockData->stateKeys.push_back(statesStr); + } + } + } + } else { + trackingBlock.clear(); + } + + // if a move over lea, we maybe loading block states + if (buffer[10] == 'm' && buffer[11] == 'o' && buffer[12] == 'v') { + std::string_view line(buffer); + // if line includes '#', then split by the comment and get the comment + size_t pos = line.find("#"); + if (pos != std::string::npos) { + auto comment = line.substr(pos + 2, line.size() - pos - 1); + + // For some reason some blocks are loaded outside of the block registry? + if (isInBlockRegistry || currentBlockData.has_value()) { + auto addressPos = comment.find(" "); + if (addressPos != std::string::npos) { + auto addressStr = comment.substr(0, addressPos); + auto addressInt = hexStr2Int(addressStr); + lastLastLoadedAddress = lastLoadedAddress; + lastLoadedAddress = addressInt; + } + } else if (isInGlobalBlock) { + // State Registration? + auto statesPos = comment.find("VanillaStates::"); + if (statesPos != std::string::npos) { + // ensure there's no + in the symbol + if (comment.find("+") != std::string::npos) { + continue; + } + auto states = comment.substr(statesPos + 15, comment.size() - statesPos - 16); + if (stringsMap.find(lastLoadedAddress) != stringsMap.end()) { + std::cout << "VanillaState\t" << states << "\t" << lastLoadedAddressAbsMovStr << "\t" + << stringsMap[lastLoadedAddress] << std::endl; + } + } + } + } + } + + // if buffer ends with a colon, then it's a new block + if (buffer[0] == '0') { + std::string_view line(buffer); + trackingBlock.clear(); + // globals initialization + if (line.find("<_GLOBAL_") != std::string::npos) { + isInGlobalBlock = true; + } else { + isInGlobalBlock = false; + } + if (line.find("::registerBlocks") != std::string::npos) { + isInBlockRegistry = true; + } else { + isInBlockRegistry = false; + if (currentBlockData.has_value()) { + blockData.push_back(currentBlockData.value()); + currentBlockData.reset(); + } + } + + // 000000000715d070 (Tag const&, int&)>: + if (line.find("StateSerializationUtils::fromNBT<") != std::string::npos) { + auto pos = line.find("StateSerializationUtils::fromNBT<"); + auto end = line.find(">", pos); + auto substr = line.substr(pos + 33, end - pos - 33); + std::cout << "StateSerializer\t" << substr << std::endl; + inStateSerializer = std::string(substr); + } else { + inStateSerializer.clear(); + } + } + ZeroMemory(buffer, bufferSize); + } + + if (!filePath.empty()) { + ((std::ifstream *)disStream)->close(); + delete disStream; + } + + // Print out the block data + for (auto &block : blockData) { + std::cout << "BlockData\t" << block.blockName << "\t" << block.blockClass << "\t" << block.breakTimeAddr << "\t"; + for (auto &state : block.stateKeys) { + std::cout << state << ","; + } + std::cout << std::endl; + } + + // Print out the state entries + for (auto &entry : stateEntries) { + std::cout << "StateEntry\t" << entry.first << "\t"; + for (auto &state : entry.second) { + std::cout << state << ","; + } + std::cout << std::endl; + } +} + +// StateHash -> integer data for this state (like number of variants) +std::map> stateVariantMap; + +void split4(std::string_view line, std::string &a, std::string &b, std::string &c, std::string &d) { + size_t pos = line.find("\t"); + if (pos != std::string::npos) { + a = std::string(line.substr(0, pos)); + size_t pos2 = line.find("\t", pos + 1); + if (pos2 != std::string::npos) { + b = std::string(line.substr(pos + 1, pos2 - pos - 1)); + size_t pos3 = line.find("\t", pos2 + 1); + if (pos3 != std::string::npos) { + c = std::string(line.substr(pos2 + 1, pos3 - pos2 - 1)); + d = std::string(line.substr(pos3 + 1, line.size() - pos3 - 1)); + } + } + } +} + +void loadStage1(std::string filePath) { + // load stage1.txt which is the output of above loadDisassembly function + std::ifstream stage1Stream(filePath, std::ios::binary); + if (!stage1Stream.is_open()) { + std::cerr << "Failed to open file: stage1.txt" << std::endl; + return; + } + // split by tabs + const int bufferSize = 1024 * 64; + char buffer[bufferSize]; + while (stage1Stream.getline(buffer, bufferSize)) { + std::string_view line(buffer); + size_t pos = line.find("\t"); + if (pos != std::string::npos) { + // VanillaState StateID StatHash StateName + // VanillaState BiteCounter 0x6bcbbe2ee1f42f72 bite_counter + // we are interested in the 2nd and 3rd columns + std::string name, id, hash, stateName; + split4(line, name, id, hash, stateName); + if (name == "VanillaState") { + stateVariantMap[hash] = {}; + } + } + } +} + +void loadDisassembly2(std::string filePath) { + // now load the disassembly again, but windows.asm which has a bit more information, like the state variants + std::istream *disStream = &std::cin; + if (!filePath.empty()) { + disStream = new std::ifstream(filePath, std::ios::binary); + if (!((std::ifstream *)disStream)->is_open()) { + std::cerr << "Failed to open file: " << filePath << std::endl; + return; + } + } + + bool inStateVariant = false; + std::string currentHash; + + const int bufferSize = 1024 * 64; + char buffer[bufferSize]{0}; + + bool inMovInstruction = false; + // 140064b38: 48 c7 05 ad 7f 95 02 mov QWORD PTR [rip+0x2957fad],0x4 # 0x1429bcaf0 + while (disStream->getline(buffer, bufferSize)) { + if (buffer[36] == 'm' && buffer[37] == 'o' && buffer[38] == 'v' && buffer[39] == ' ') { + inMovInstruction = true; + } else if (buffer[36] != ' ' && buffer[36] != '\0') { + // if the instruction is not a continuation of the previous instruction + inMovInstruction = false; + } + // now we are looking for the state variants... first look for movabs + if (buffer[36] == 'm' && buffer[37] == 'o' && buffer[38] == 'v' && buffer[39] == 'a' && buffer[40] == 'b' && + buffer[41] == 's') { + std::string_view line(buffer); + + for (auto &entry : stateVariantMap) { + size_t pos = line.find(entry.first); + if (pos != std::string::npos) { + // we found the state hash, now we need to find the number of variants + currentHash = entry.first; + } + } + } + + // 140064b3f: 04 00 00 00 + if (inMovInstruction && buffer[36] == '\0' && currentHash.size() > 0) { + // this instruction is not an instruction but a continuation of the previous mov instruction + // this should contain the number of state variants + std::string_view line(buffer); + // split by the colon + size_t pos = line.find(":"); + if (pos != std::string::npos) { + auto hexStr = line.substr(pos + 1, line.size() - pos); + unsigned int value = hexStr2IntLE(hexStr); + // std::cout << "Found state data: [" << hexStr << "] (" << value << ")" << std::endl; + stateVariantMap[currentHash].push_back(value); + // max is 3 entries + if (stateVariantMap[currentHash].size() == 3) { + currentHash.clear(); + } + } + } + ZeroMemory(buffer, bufferSize); + } + + if (!filePath.empty()) { + ((std::ifstream *)disStream)->close(); + delete disStream; + } + + // print out the state variant data + for (auto &entry : stateVariantMap) { + std::cout << "StateVariantData\t" << entry.first << "\t"; + for (auto &value : entry.second) { + std::cout << value << ","; + } + std::cout << std::endl; + } +} + +int main(int argc, char **argv) { + if (argc < 3) { + std::cerr << "Usage: disa -s1 [dis]" << std::endl; + std::cerr << "Usage: disa -s2 [dis]" << std::endl; + return 1; + } + std::string stage = argv[1]; + std::string file = argv[2]; + std::string disFile; + if (argc > 3) { + disFile = argv[3]; + } + std::cout << "Stage: " << stage << std::endl; + if (disFile.empty()) { + std::cout << "(waiting for stdin)" << std::endl; + } + if (stage == "-s1") { + loadStrings(file); + loadDisassembly(disFile); + } else if (stage == "-s2") { + loadStage1(file); + loadDisassembly2(disFile); + } + return 0; +} diff --git a/.github/helper-bot/download.js b/.github/helper-bot/download.js new file mode 100644 index 00000000..89955ed8 --- /dev/null +++ b/.github/helper-bot/download.js @@ -0,0 +1,28 @@ +const htmlURL = 'https://www.minecraft.net/en-us/download/server/bedrock' + +async function getLatestVersions () { + const html = await fetch(htmlURL).then(res => res.text()) + // Find ' Download ' + const links = [...html.matchAll(/a href="(.*?)" /g)].map(match => match[1]) + + function forOS (os) { + const url = links.find(link => link.includes(os + '/')) + if (!url) return null + const version4 = url.match(/bedrock-server-(\d+\.\d+\.\d+\.\d+)\.zip/)[1] + const version3 = version4.split('.').slice(0, 3).join('.') + return { version4, version3, url } + } + + return { + linux: forOS('linux'), + windows: forOS('win'), + macos: forOS('osx'), + preview: { + linux: forOS('linux-preview'), + windows: forOS('win-preview'), + macos: forOS('osx-preview') + } + } +} + +getLatestVersions().then(console.log).catch(console.error) \ No newline at end of file diff --git a/.github/helper-bot/index.js b/.github/helper-bot/index.js index 5b76b744..b683a0c2 100644 --- a/.github/helper-bot/index.js +++ b/.github/helper-bot/index.js @@ -43,11 +43,9 @@ ${commitData} (I will close this issue automatically if "${result.version}" is added to index.d.ts on "master" and there are no X's below) - - + + +
Name${result.version}
Protocol ID${protocolVersion}
----- diff --git a/.github/helper-bot/pdba.cpp b/.github/helper-bot/pdba.cpp new file mode 100644 index 00000000..d844e9c5 --- /dev/null +++ b/.github/helper-bot/pdba.cpp @@ -0,0 +1,42 @@ +#include +#include +#include + +void loadDump() { + std::string line; + while (std::getline(std::cin, line)) { + auto pos = line.find("`?"); + if (pos != std::string::npos) { + std::string mangledName = line.substr(pos + 1, line.size() - pos - 2); + std::string demangled = llvm::demangle(mangledName); + + if (mangledName.find("VanillaStates") == std::string::npos) { + continue; + } + auto vanillaStatesPos = demangled.find("VanillaStates::"); + if (vanillaStatesPos == std::string::npos) { + continue; + } + + auto vanillaStateName = demangled.substr(vanillaStatesPos + 15); + if (demangled.find("Variant") != std::string::npos) { + std::cout << "VST " << "\t" << vanillaStateName << "\t" << "int" << std::endl; + } else if (demangled.find("Variant") != std::string::npos) { + std::cout << "VST " << "\t" << vanillaStateName << "\t" << "bool" << std::endl; + } else { + std::cout << "VST " << "\t" << vanillaStateName << "\t" << "string" << std::endl; + } + } + } +} + +/* +char *microsoftDemangle(const char *mangled_name, size_t *n_read, char *buf, + size_t *n_buf, int *status, + MSDemangleFlags Flags = MSDF_None); +*/ + +int main() { + loadDump(); + return 0; +} diff --git a/test/vanilla.js b/test/vanilla.js index 55758d99..5323cd95 100644 --- a/test/vanilla.js +++ b/test/vanilla.js @@ -46,7 +46,6 @@ async function test (version) { client.queue('tick_sync', { request_time: BigInt(Date.now()), response_time: BigInt(Date.now()) }) }, 200) - console.log('Awaiting join') client.on('spawn', () => { From 856494ed13e44b7c7f4ea89368be19ff48cc16e6 Mon Sep 17 00:00:00 2001 From: extremeheat Date: Sun, 31 Mar 2024 18:11:26 -0400 Subject: [PATCH 04/50] helper-bot updates --- .github/helper-bot/disa.cpp | 12 ++-- .github/helper-bot/download.js | 2 +- .github/helper-bot/index.js | 12 +--- .github/helper-bot/package.json | 5 ++ .github/helper-bot/update1.js | 68 ++++++++++++++++++ .github/helper-bot/update2.js | 123 ++++++++++++++++++++++++++++++++ 6 files changed, 202 insertions(+), 20 deletions(-) create mode 100644 .github/helper-bot/update1.js create mode 100644 .github/helper-bot/update2.js diff --git a/.github/helper-bot/disa.cpp b/.github/helper-bot/disa.cpp index 970cb1b0..787a1475 100644 --- a/.github/helper-bot/disa.cpp +++ b/.github/helper-bot/disa.cpp @@ -175,7 +175,6 @@ void loadDisassembly(std::string filePath) { size_t pos = line.find("#"); if (pos != std::string::npos) { - // std::cout << trackingBlock << " " << buffer << std::endl; auto comment = line.substr(pos + 2, line.size() - pos - 1); // above bounds to skip the comment+space // split the comment by the first space @@ -227,7 +226,7 @@ void loadDisassembly(std::string filePath) { if (pos != std::string::npos) { auto comment = line.substr(pos + 2, line.size() - pos - 1); - // For some reason some blocks are loaded outside of the block registry? + // Some reason some blocks are loaded outside of the block registry if (isInBlockRegistry || currentBlockData.has_value()) { auto addressPos = comment.find(" "); if (addressPos != std::string::npos) { @@ -332,10 +331,10 @@ void split4(std::string_view line, std::string &a, std::string &b, std::string & } void loadStage1(std::string filePath) { - // load stage1.txt which is the output of above loadDisassembly function + // load stage1 which is the output of above loadDisassembly function std::ifstream stage1Stream(filePath, std::ios::binary); if (!stage1Stream.is_open()) { - std::cerr << "Failed to open file: stage1.txt" << std::endl; + std::cerr << "Failed to open file: " << filePath << std::endl; return; } // split by tabs @@ -345,7 +344,7 @@ void loadStage1(std::string filePath) { std::string_view line(buffer); size_t pos = line.find("\t"); if (pos != std::string::npos) { - // VanillaState StateID StatHash StateName + // VanillaState StateID StateHash StateName // VanillaState BiteCounter 0x6bcbbe2ee1f42f72 bite_counter // we are interested in the 2nd and 3rd columns std::string name, id, hash, stateName; @@ -358,7 +357,6 @@ void loadStage1(std::string filePath) { } void loadDisassembly2(std::string filePath) { - // now load the disassembly again, but windows.asm which has a bit more information, like the state variants std::istream *disStream = &std::cin; if (!filePath.empty()) { disStream = new std::ifstream(filePath, std::ios::binary); @@ -407,7 +405,6 @@ void loadDisassembly2(std::string filePath) { if (pos != std::string::npos) { auto hexStr = line.substr(pos + 1, line.size() - pos); unsigned int value = hexStr2IntLE(hexStr); - // std::cout << "Found state data: [" << hexStr << "] (" << value << ")" << std::endl; stateVariantMap[currentHash].push_back(value); // max is 3 entries if (stateVariantMap[currentHash].size() == 3) { @@ -423,7 +420,6 @@ void loadDisassembly2(std::string filePath) { delete disStream; } - // print out the state variant data for (auto &entry : stateVariantMap) { std::cout << "StateVariantData\t" << entry.first << "\t"; for (auto &value : entry.second) { diff --git a/.github/helper-bot/download.js b/.github/helper-bot/download.js index 89955ed8..b77449ac 100644 --- a/.github/helper-bot/download.js +++ b/.github/helper-bot/download.js @@ -25,4 +25,4 @@ async function getLatestVersions () { } } -getLatestVersions().then(console.log).catch(console.error) \ No newline at end of file +getLatestVersions().then(console.log).catch(console.error) diff --git a/.github/helper-bot/index.js b/.github/helper-bot/index.js index b683a0c2..14a25781 100644 --- a/.github/helper-bot/index.js +++ b/.github/helper-bot/index.js @@ -13,7 +13,6 @@ const changelogURL = 'https://feedback.minecraft.net/hc/en-us/sections/360001186 function buildFirstIssue (title, result, externalPatches) { let commitData = '' - let protocolVersion = '?' const date = new Date(result.currentVersionReleaseDate).toUTCString() for (const name in externalPatches) { @@ -25,7 +24,6 @@ function buildFirstIssue (title, result, externalPatches) { if (diff) commitData += `\n**[See the diff between *${result.currentVersionReleaseDate}* and now](${diff})**\n` else commitData += '\n(No changes so far)\n' } - try { protocolVersion = getProtocolVersion() } catch (e) { console.log(e) } return { title, @@ -73,19 +71,12 @@ function getCommitsInRepo (repo, containing, since) { if (commits.length) { const head = commits[0].sha const tail = commits[commits.length - 1].sha - return [relevant, `https://github.com/${repo}/compare/${tail}..${head}`] + return [relevant, `https://github.com/${repo}/compare/${tail}..${head}`] } } return [relevant] } -function getProtocolVersion () { - if (!fs.existsSync('./ProtocolInfo.php')) cp.execSync('curl -LO https://raw.githubusercontent.com/pmmp/PocketMine-MP/stable/src/pocketmine/network/mcpe/protocol/ProtocolInfo.php', { stdio: 'inherit', shell: true }) - const currentApi = fs.readFileSync('./ProtocolInfo.php', 'utf-8') - const [, latestProtocolVersion] = currentApi.match(/public const CURRENT_PROTOCOL = (\d+);/) - return latestProtocolVersion -} - async function fetchLatest () { if (!fs.existsSync('./results.json')) cp.execSync(`curl -L "${latestVesionEndpoint}" -o results.json`, { stdio: 'inherit', shell: true }) const json = require('./results.json') @@ -111,7 +102,6 @@ async function fetchLatest () { return } - if (issueStatus.isClosed) { // We already made an issue, but someone else already closed it, don't do anything else console.log('I already made an issue, but it was closed') diff --git a/.github/helper-bot/package.json b/.github/helper-bot/package.json index 658d5db8..230a8945 100644 --- a/.github/helper-bot/package.json +++ b/.github/helper-bot/package.json @@ -1,4 +1,9 @@ { + "description": "Minecraft Bedrock automatic update system", + "scripts": { + "test": "standard", + "fix": "standard --fix" + }, "dependencies": { "gh-helpers": "^0.2.0" } diff --git a/.github/helper-bot/update1.js b/.github/helper-bot/update1.js new file mode 100644 index 00000000..c2337ada --- /dev/null +++ b/.github/helper-bot/update1.js @@ -0,0 +1,68 @@ +const fs = require('fs') +const core = require('@actions/core') +const github = require('gh-helpers')() +const bedrock = require('bedrock-protocol') +const bedrockServer = require('minecraft-bedrock-server') +const path = require('path') + +async function tryConnect (opts) { + const client = bedrock.createClient({ + host: 'localhost', + port: 19132, + username: 'test', + offline: true + }) + client.on('connect_allowed', () => { Object.assign(client.options, opts) }) + + return new Promise((resolve, reject) => { + const collected = {} + const forCollection = ['start_game', 'available_commands'] + for (const packet of forCollection) { + client.once(packet, (data) => { + collected[packet] = data + fs.writeFileSync(path.join(__dirname, '/collected.json'), JSON.stringify(collected, null, 2)) + }) + if (Object.keys(collected).length === forCollection.length) { + resolve(collected) + } + } + + setTimeout(() => { + if (Object.keys(collected).length !== forCollection.length) { + reject(Error('Unable to collect all packets')) + } + }, 1000 * 60 * 6) + }) +} + +async function main (inputUpdateVer, inputIssueNo) { + const issue = await github.findIssue({ number: inputIssueNo }) + const latestServers = await bedrockServer.getLatestVersions() + let updatedBody = issue.body + const serverVersion = latestServers.linux.version3 + if (serverVersion !== inputUpdateVer) { + updatedBody = updatedBody.replace('', `Server version${serverVersion}`) + } + const root = path.resolve(path.join(__dirname, '../../tools')) + const serverPath = root + '/bds-' + serverVersion + console.log('Server version', serverVersion, root, 'Server path', serverPath) + core.setOutput('serverVersion', serverVersion) + core.setOutput('serverPath', serverPath) + core.setOutput('serverBin', serverPath + '/bedrock_server_symbols.debug') + const handle = await bedrockServer.startServerAndWait(serverVersion, 60000, { root }) + const pong = await bedrock.ping({ host: 'localhost', port: 19132 }) + updatedBody = updatedBody.replace('', `Protocol ID${pong.protocol} (${pong.version})`) + try { + await tryConnect({ protocolVersion: pong.protocol }) + updatedBody = updatedBody.replace('', '') + } catch (e) { + console.error(e) + updatedBody = updatedBody.replace('', 'Partly Already CompatibleNO') + } + fs.writeFileSync(path.join(__dirname, '/updatedBody.md', updatedBody)) + await github.updateIssue({ number: inputIssueNo, body: updatedBody }) + handle.kill() +} + +// main(process.env.UPDATE_VERSION, process.env.ISSUE_NUMBER) +main(process.env.UPDATE_VERSION, process.env.ISSUE_NUMBER) diff --git a/.github/helper-bot/update2.js b/.github/helper-bot/update2.js new file mode 100644 index 00000000..ce1e490b --- /dev/null +++ b/.github/helper-bot/update2.js @@ -0,0 +1,123 @@ +/* eslint-disable no-tabs */ +const { join } = require('path') +const fs = require('fs') + +/* +Hex dump of section '.rodata': + 0x00ba0080 00004a42 000000c0 17b7d138 9a99193e ..JB.......8...> + 0x00ba0090 00008043 000020c2 c3f5e8be 0000003c ...C.. ........< +*/ + +function loadRoDataHexDump (filePath) { + const rodataHexDump = fs.readFileSync(filePath, 'latin1') + const hexDumpLines = rodataHexDump.split('\n') + const buffer = Buffer.alloc(hexDumpLines.length * 16) + let initialBufferIx + let bufferIx = 0 + for (const line of hexDumpLines) { + if (!line.startsWith(' ')) continue + initialBufferIx ||= line.slice(2, 12) + const data = line.slice(13, 48).replaceAll(' ', '') + bufferIx += buffer.write(data, bufferIx, 'hex') + } + const initialBufferOffset = parseInt(initialBufferIx, 16) + + function readStringNTFromHexDump (addr) { + const addrNum = typeof addr === 'string' ? parseInt(addr, 16) : addr + const startIx = addrNum - initialBufferOffset + const endIx = buffer.indexOf(0, startIx) + if (endIx === -1) return null + return buffer.toString('latin1', startIx, endIx) + } + function readFloatLEFromHexDump (addr) { + const addrNum = typeof addr === 'string' ? parseInt(addr, 16) : addr + const startIx = addrNum - initialBufferOffset + return buffer.readFloatLE(startIx) + } + function readDoubleLEFromHexDump (addr) { + const addrNum = typeof addr === 'string' ? parseInt(addr, 16) : addr + const startIx = addrNum - initialBufferOffset + return buffer.readDoubleLE(startIx) + } + function readIntLEFromHexDump (addr) { + const addrNum = typeof addr === 'string' ? parseInt(addr, 16) : addr + const startIx = addrNum - initialBufferOffset + return buffer.readInt32LE(startIx) + } + function readI64LEFromHexDump (addr) { + const addrNum = typeof addr === 'string' ? parseInt(addr, 16) : addr + const startIx = addrNum - initialBufferOffset + return buffer.readBigInt64LE(startIx) + } + return { buffer, initialBufferOffset, readStringNTFromHexDump, readFloatLEFromHexDump, readDoubleLEFromHexDump, readIntLEFromHexDump, readI64LEFromHexDump } +} + +function writeStringsTSV (rodataDump, stringsFile) { + const { readStringNTFromHexDump } = loadRoDataHexDump(rodataDump, stringsFile) + + let result = '' + const strings = fs.readFileSync(stringsFile, 'utf8') + for (let i = 0; i < strings.length; i++) { + const endIx = strings.indexOf('\n', i) + // split the sub string by spaces + const subStr = strings.substring(i, endIx) + const split = subStr.split(' ') + const address = split.find(s => s.startsWith('0x')) + const contents = subStr.split('ascii')[1]?.trim() + if (!contents || !contents.match(/^[a-zA-Z0-9_:]+$/) || contents.length > 64) { + if (subStr.includes('utf16le')) { + // sigh, some ascii strings are written as utf16le + const contents16 = subStr.split('utf16le')[1]?.trim() + for (let j = 0; j < contents16.length; j += 1) { + // write each char one by one as 1 length string + // const char = contents16[j] + const newAddress = `0x${(parseInt(address, 16) + (j * 2)).toString(16).padStart(8, '0')}` + result += `# fromU16LE ${address}\n` + const ntString = readStringNTFromHexDump(newAddress) + if (ntString && ntString.match(/^[a-zA-Z0-9_:]+$/) && ntString.length < 64) { + result += `${newAddress}\t${ntString}\n` + } + } + } + } else { + result += `${address}\t${contents}\n` + } + i = endIx + } + fs.writeFileSync('strings.tsv', result) +} + +if (process.argv.length < 2) { + console.log('Usage: node extract.js ') + process.exit(1) +} + +function postProc (rodataDump, s1file) { + const { readFloatLEFromHexDump } = loadRoDataHexDump(rodataDump) + const stage2 = fs.readFileSync(s1file, 'latin1') + for (const line of stage2.split('\n')) { + const slices = line.split('\t') + if (slices[0] === 'BlockData') { + // console.log(slices) + try { + const float = readFloatLEFromHexDump(parseInt(slices[3])) + console.log('BlockData', slices[1], slices[2], float) + } catch (e) { + console.log('BlockData', slices[1], slices[2], 0) + } + } + } +} + +const stage = process.argv[2] +if (stage === 'stage0') { + writeStringsTSV( + join(__dirname, 'rodata.txt'), + join(__dirname, 'strings.txt') + ) +} else if (stage === 'stage3') { + postProc( + join(__dirname, 'rodata.txt'), + join(__dirname, 'stage1.txt') + ) +} From ac60885c418d10aecffc3c4e92fecf16dbe5a78a Mon Sep 17 00:00:00 2001 From: extremeheat Date: Sun, 31 Mar 2024 19:15:08 -0400 Subject: [PATCH 05/50] update helper bot workflow --- .github/helper-bot/package.json | 2 +- .github/helper-bot/update1.js | 8 ++++ .github/helper-bot/update2.js | 4 +- .github/helper-bot/update3.js | 28 +++++++++++++ .github/workflows/update-helper.yml | 64 ++++++++++++++++++++++++++++- 5 files changed, 102 insertions(+), 4 deletions(-) create mode 100644 .github/helper-bot/update3.js diff --git a/.github/helper-bot/package.json b/.github/helper-bot/package.json index 230a8945..ae068535 100644 --- a/.github/helper-bot/package.json +++ b/.github/helper-bot/package.json @@ -5,6 +5,6 @@ "fix": "standard --fix" }, "dependencies": { - "gh-helpers": "^0.2.0" + "gh-helpers": "^0.2.1" } } diff --git a/.github/helper-bot/update1.js b/.github/helper-bot/update1.js index c2337ada..7f668b10 100644 --- a/.github/helper-bot/update1.js +++ b/.github/helper-bot/update1.js @@ -62,6 +62,14 @@ async function main (inputUpdateVer, inputIssueNo) { fs.writeFileSync(path.join(__dirname, '/updatedBody.md', updatedBody)) await github.updateIssue({ number: inputIssueNo, body: updatedBody }) handle.kill() + console.log('✅ Finished working with Linux server binary') + console.log('Working now on Windows') + const winPath = serverPath.replace('bds-', 'bds-win-') + await bedrockServer.downloadServer(latestServers.windows.version3, winPath) + core.setOutput('serverWinPath', winPath) + core.setOutput('serverWinBin', winPath + '/bedrock_server.exe') + core.setOutput('serverWinPdb', winPath + '/bedrock_server.pdb') + console.log('✅ Finished working with Windows server binary') } // main(process.env.UPDATE_VERSION, process.env.ISSUE_NUMBER) diff --git a/.github/helper-bot/update2.js b/.github/helper-bot/update2.js index ce1e490b..fcac8236 100644 --- a/.github/helper-bot/update2.js +++ b/.github/helper-bot/update2.js @@ -110,12 +110,12 @@ function postProc (rodataDump, s1file) { } const stage = process.argv[2] -if (stage === 'stage0') { +if (stage === '-s0') { writeStringsTSV( join(__dirname, 'rodata.txt'), join(__dirname, 'strings.txt') ) -} else if (stage === 'stage3') { +} else if (stage === '-s3') { postProc( join(__dirname, 'rodata.txt'), join(__dirname, 'stage1.txt') diff --git a/.github/helper-bot/update3.js b/.github/helper-bot/update3.js new file mode 100644 index 00000000..ab4a4dea --- /dev/null +++ b/.github/helper-bot/update3.js @@ -0,0 +1,28 @@ +const fs = require('fs') +const github = require('gh-helpers')() + +async function main () { + const stages = ['stage1.txt', 'stage2.txt', 'stage3.txt', 'stage4.txt'] + const allStages = fs.createWriteStream('merged.txt') + for (const stage of stages) { + allStages.write(fs.readFileSync(stage, 'latin1')) + } + allStages.end() + const artifact = await github.artifacts.createTextArtifact('updatorData', { + content: fs.readFileSync('merged.txt', 'latin1') + }) + console.log('Created artifact', artifact) + const dispatch = await github.sendWorkflowDispatch({ + repo: 'llm-services', + workflow: 'dispatch.yml', + branch: 'main', + inputs: { + repoData: await github.getRepoDetails(), + artifactId: artifact.id, + artifactSize: artifact.size + } + }) + console.log('Sent workflow dispatch', dispatch) +} + +main() diff --git a/.github/workflows/update-helper.yml b/.github/workflows/update-helper.yml index 551746d8..48f87f13 100644 --- a/.github/workflows/update-helper.yml +++ b/.github/workflows/update-helper.yml @@ -8,6 +8,9 @@ jobs: helper: name: update-checker runs-on: ubuntu-latest + outputs: + updateVersion: ${{ steps.helperRun.outputs.updateVersion }} + issueNumber: ${{ steps.helperRun.outputs.issueNumber }} steps: - name: Checkout repository uses: actions/checkout@master @@ -20,7 +23,66 @@ jobs: working-directory: .github/helper-bot # The env vars contain the relevant trigger information, so we don't need to pass it - name: Runs helper + id: helperRun run: node index.js working-directory: .github/helper-bot env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.PAT_PASSWORD }} + # Above step sets output of updateVersion, issueNumber with actions/core + + update: + name: Update + needs: helper + runs-on: ubuntu-latest + # run if updateVersion exists + if: ${{ needs.helper.outputs.updateVersion }} + steps: + - name: Checkout repository + uses: actions/checkout@master + - name: Set up Node.js + uses: actions/setup-node@master + with: + node-version: 18.0.0 + - name: Compile disa + run: clang++ -std=c++20 -O3 -o disa.exe disa.cpp + working-directory: .github/helper-bot + - name: Compile pdba + run: clang++-14 -g `llvm-config-14 --cxxflags --ldflags --libs` -o pdb.exe pdba.cpp + - name: Install radare2 + run: sudo apt-get install radare2 + - name: Install + run: npm install gh-helpers + # This step sets the "serverVersion" and "serverPath" outputs + - name: Update Step 1 + id: update1Run + run: cd .github/helper-bot && node update1.js + # Pass the output from the helper job to the update job + env: + GITHUB_TOKEN: ${{ secrets.PAT_PASSWORD }} + UPDATE_VERSION: ${{ needs.helper.outputs.updateVersion }} + ISSUE_NUMBER: ${{ needs.helper.outputs.issueNumber }} + - name: Dump .rodata strings + run: rabin2 -N 1 -z ${{ steps.update1Run.outputs.serverBin }} > .github/helper-bot/strings.txt + # radare2 -a x86 -b elf -qc 'izz~ro' ${{ steps.update1Run.outputs.serverBin }} > ./github/helper-bot/botstr.txt + - name: Dump .rodata + run: readelf -x .rodata ${{ steps.update1Run.outputs.serverBin }} > .github/helper-bot/rodata.txt + - name: Proc strings (Stage 0) + run: node update2.js -s0 > strings.tsv + working-directory: .github/helper-bot + # objdump the server binary .text, using "serverPath" output from the previous step + - name: Process Linux server (Stage 1) + run: objdump -d --demangle --no-show-raw-insn --debugging ${{ steps.update1Run.outputs.serverBin }} | .github/helper-bot/disa.exe -s1 .github/helper-bot/strings.txt > .github/helper-bot/stage1.txt + - name: Process Windows bin (Stage 2) + run: objdump -d --demangle -M intel ${{ steps.update1Run.outputs.serverWinBin }} | .github/helper-bot/disa.exe -s2 .github/helper-bot/stage1.txt > .github/helper-bot/stage2.txt + - name: Resolve numbers (Stage 3) + run : node update2.js -s3 + working-directory: .github/helper-bot + - name: Resolve PDB (Stage 4) + run: llvm-pdbutil dump --all ${{ steps.update1Run.outputs.serverWinPdb }} | .github/helper-bot/pdba.exe > .github/helper-bot/stage4.txt + - name: Aggregate and finish + run: node update3.js + working-directory: .github/helper-bot + env: + GITHUB_TOKEN: ${{ secrets.PAT_PASSWORD }} + UPDATE_VERSION: ${{ needs.helper.outputs.updateVersion }} + ISSUE_NUMBER: ${{ needs.helper.outputs.issueNumber }} \ No newline at end of file From cf29575d3c7301e72c82c0b98c562e792fa2aadd Mon Sep 17 00:00:00 2001 From: extremeheat Date: Sun, 31 Mar 2024 20:07:56 -0400 Subject: [PATCH 06/50] update tools --- tools/dumpPackets.js | 2 +- tools/genPacketDumps.js | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/dumpPackets.js b/tools/dumpPackets.js index 47cc30bb..d276cd88 100644 --- a/tools/dumpPackets.js +++ b/tools/dumpPackets.js @@ -2,7 +2,7 @@ // uses the same format as prismarine-packet-dumper const assert = require('assert') const fs = require('fs') -const vanillaServer = require('../tools/startVanillaServer') +const vanillaServer = require('minecraft-bedrock-server') const { Client } = require('../src/client') const { serialize, waitFor } = require('../src/datatypes/util') const { CURRENT_VERSION } = require('../src/options') diff --git a/tools/genPacketDumps.js b/tools/genPacketDumps.js index 2e6420ff..8530e978 100644 --- a/tools/genPacketDumps.js +++ b/tools/genPacketDumps.js @@ -1,12 +1,12 @@ // Collect sample packets needed for `serverTest.js` // process.env.DEBUG = 'minecraft-protocol' const fs = require('fs') -const vanillaServer = require('../tools/startVanillaServer') +const vanillaServer = require('minecraft-bedrock-server') const { Client } = require('../src/client') const { serialize, waitFor, getFiles } = require('../src/datatypes/util') const { CURRENT_VERSION } = require('../src/options') const { join } = require('path') -const { getPort } = require('../test/util') +const { getPort } = require('../test/util/util') function hasDumps (version) { const root = join(__dirname, `../data/${version}/sample/packets/`) @@ -25,7 +25,7 @@ async function dump (version, force = true) { const [port, v6] = [await getPort(), await getPort()] console.log('Starting dump server', version) - const handle = await vanillaServer.startServerAndWait2(version || CURRENT_VERSION, 1000 * 120, { 'server-port': port, 'server-portv6': v6 }) + const handle = await vanillaServer.startServerAndWait2(version || CURRENT_VERSION, 1000 * 60 * 3, { 'server-port': port, 'server-portv6': v6 }) console.log('Started dump server', version) const client = new Client({ From 4e8c26039a8a299ec0a95e1da4e7f3ea8cd5a4c9 Mon Sep 17 00:00:00 2001 From: extremeheat Date: Sun, 31 Mar 2024 22:21:03 -0400 Subject: [PATCH 07/50] update tests --- test/proxy.js | 2 +- tools/dumpPackets.js | 5 ++++- tools/genPacketDumps.js | 6 +++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/test/proxy.js b/test/proxy.js index 345e89fc..09506c76 100644 --- a/test/proxy.js +++ b/test/proxy.js @@ -1,6 +1,6 @@ const { createClient, Server, Relay } = require('bedrock-protocol') const { sleep, waitFor } = require('../src/datatypes/util') -const { getPort } = require('./util') +const { getPort } = require('./util/util') function proxyTest (version, raknetBackend = 'raknet-native', timeout = 1000 * 40) { console.log('with raknet backend', raknetBackend) diff --git a/tools/dumpPackets.js b/tools/dumpPackets.js index d276cd88..0d33f2e6 100644 --- a/tools/dumpPackets.js +++ b/tools/dumpPackets.js @@ -16,7 +16,10 @@ async function dump (version) { const random = ((Math.random() * 100) | 0) const port = 19130 + random - const handle = await vanillaServer.startServerAndWait(version || CURRENT_VERSION, 1000 * 120, { 'server-port': port }) + const handle = await vanillaServer.startServerAndWait(version || CURRENT_VERSION, 1000 * 120, { + 'server-port': port, + root: __dirname + }) console.log('Started server') const client = new Client({ diff --git a/tools/genPacketDumps.js b/tools/genPacketDumps.js index 8530e978..5415fa67 100644 --- a/tools/genPacketDumps.js +++ b/tools/genPacketDumps.js @@ -25,7 +25,11 @@ async function dump (version, force = true) { const [port, v6] = [await getPort(), await getPort()] console.log('Starting dump server', version) - const handle = await vanillaServer.startServerAndWait2(version || CURRENT_VERSION, 1000 * 60 * 3, { 'server-port': port, 'server-portv6': v6 }) + const handle = await vanillaServer.startServerAndWait2(version || CURRENT_VERSION, 1000 * 60 * 3, { + 'server-port': port, + 'server-portv6': v6, + root: __dirname + }) console.log('Started dump server', version) const client = new Client({ From 79aa0f3a1b11eb2be11308d47da782e22833c08a Mon Sep 17 00:00:00 2001 From: extremeheat Date: Mon, 1 Apr 2024 07:24:12 +0000 Subject: [PATCH 08/50] helperbot update --- .github/helper-bot/.gitignore | 5 +++++ .github/helper-bot/update2.js | 7 +++++-- .github/workflows/update-helper.yml | 6 +++--- .gitpod.DockerFile | 8 -------- 4 files changed, 13 insertions(+), 13 deletions(-) create mode 100644 .github/helper-bot/.gitignore delete mode 100644 .gitpod.DockerFile diff --git a/.github/helper-bot/.gitignore b/.github/helper-bot/.gitignore new file mode 100644 index 00000000..7d9c40f6 --- /dev/null +++ b/.github/helper-bot/.gitignore @@ -0,0 +1,5 @@ +bds-* +*.exe +stage* +rodata.txt +strings.* \ No newline at end of file diff --git a/.github/helper-bot/update2.js b/.github/helper-bot/update2.js index fcac8236..22505b9e 100644 --- a/.github/helper-bot/update2.js +++ b/.github/helper-bot/update2.js @@ -95,18 +95,21 @@ if (process.argv.length < 2) { function postProc (rodataDump, s1file) { const { readFloatLEFromHexDump } = loadRoDataHexDump(rodataDump) const stage2 = fs.readFileSync(s1file, 'latin1') + let result = '' for (const line of stage2.split('\n')) { const slices = line.split('\t') if (slices[0] === 'BlockData') { // console.log(slices) try { const float = readFloatLEFromHexDump(parseInt(slices[3])) - console.log('BlockData', slices[1], slices[2], float) + console.log('BlockExtraData', slices[1], float) + result += `BlockExtraData\t${slices[1]}\t${float}\n` } catch (e) { - console.log('BlockData', slices[1], slices[2], 0) + result += `BlockExtraData\t${slices[1]}\t${0}\n` } } } + fs.writeFileSync('stage3.txt', result) } const stage = process.argv[2] diff --git a/.github/workflows/update-helper.yml b/.github/workflows/update-helper.yml index 48f87f13..0fbad877 100644 --- a/.github/workflows/update-helper.yml +++ b/.github/workflows/update-helper.yml @@ -47,9 +47,9 @@ jobs: run: clang++ -std=c++20 -O3 -o disa.exe disa.cpp working-directory: .github/helper-bot - name: Compile pdba - run: clang++-14 -g `llvm-config-14 --cxxflags --ldflags --libs` -o pdb.exe pdba.cpp + run: clang++-14 -g `llvm-config-14 --cxxflags --ldflags --libs` -o pdba.exe pdba.cpp - name: Install radare2 - run: sudo apt-get install radare2 + run: sudo apt-get install radare2 -y - name: Install run: npm install gh-helpers # This step sets the "serverVersion" and "serverPath" outputs @@ -67,7 +67,7 @@ jobs: - name: Dump .rodata run: readelf -x .rodata ${{ steps.update1Run.outputs.serverBin }} > .github/helper-bot/rodata.txt - name: Proc strings (Stage 0) - run: node update2.js -s0 > strings.tsv + run: node update2.js -s0 working-directory: .github/helper-bot # objdump the server binary .text, using "serverPath" output from the previous step - name: Process Linux server (Stage 1) diff --git a/.gitpod.DockerFile b/.gitpod.DockerFile deleted file mode 100644 index 061bf596..00000000 --- a/.gitpod.DockerFile +++ /dev/null @@ -1,8 +0,0 @@ -FROM gitpod/workspace-full:latest - -RUN bash -c ". .nvm/nvm.sh \ - && nvm install 14 \ - && nvm use 14 \ - && nvm alias default 14" - -RUN echo "nvm use default &>/dev/null" >> ~/.bashrc.d/51-nvm-fix \ No newline at end of file From f80a42ff1bd5f2c75f222d28e739f1994446798d Mon Sep 17 00:00:00 2001 From: extremeheat Date: Mon, 1 Apr 2024 09:09:47 +0000 Subject: [PATCH 09/50] update workflow --- .github/helper-bot/update1.js | 2 ++ .github/helper-bot/update3.js | 4 +++- .github/workflows/update-helper.yml | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/helper-bot/update1.js b/.github/helper-bot/update1.js index 7f668b10..776dafe5 100644 --- a/.github/helper-bot/update1.js +++ b/.github/helper-bot/update1.js @@ -74,3 +74,5 @@ async function main (inputUpdateVer, inputIssueNo) { // main(process.env.UPDATE_VERSION, process.env.ISSUE_NUMBER) main(process.env.UPDATE_VERSION, process.env.ISSUE_NUMBER) + +// TODO: determine if protocol version has changed before doing anything \ No newline at end of file diff --git a/.github/helper-bot/update3.js b/.github/helper-bot/update3.js index ab4a4dea..4bc4b6b4 100644 --- a/.github/helper-bot/update3.js +++ b/.github/helper-bot/update3.js @@ -19,7 +19,9 @@ async function main () { inputs: { repoData: await github.getRepoDetails(), artifactId: artifact.id, - artifactSize: artifact.size + artifactSize: artifact.size, + updateVersion: process.env.UPDATE_VERSION, + issueNo: process.env.ISSUE_NUMBER } }) console.log('Sent workflow dispatch', dispatch) diff --git a/.github/workflows/update-helper.yml b/.github/workflows/update-helper.yml index 0fbad877..6b870ea3 100644 --- a/.github/workflows/update-helper.yml +++ b/.github/workflows/update-helper.yml @@ -48,6 +48,7 @@ jobs: working-directory: .github/helper-bot - name: Compile pdba run: clang++-14 -g `llvm-config-14 --cxxflags --ldflags --libs` -o pdba.exe pdba.cpp + working-directory: .github/helper-bot - name: Install radare2 run: sudo apt-get install radare2 -y - name: Install @@ -69,7 +70,6 @@ jobs: - name: Proc strings (Stage 0) run: node update2.js -s0 working-directory: .github/helper-bot - # objdump the server binary .text, using "serverPath" output from the previous step - name: Process Linux server (Stage 1) run: objdump -d --demangle --no-show-raw-insn --debugging ${{ steps.update1Run.outputs.serverBin }} | .github/helper-bot/disa.exe -s1 .github/helper-bot/strings.txt > .github/helper-bot/stage1.txt - name: Process Windows bin (Stage 2) @@ -77,7 +77,7 @@ jobs: - name: Resolve numbers (Stage 3) run : node update2.js -s3 working-directory: .github/helper-bot - - name: Resolve PDB (Stage 4) + - name: Resolve PDB symbols (Stage 4) run: llvm-pdbutil dump --all ${{ steps.update1Run.outputs.serverWinPdb }} | .github/helper-bot/pdba.exe > .github/helper-bot/stage4.txt - name: Aggregate and finish run: node update3.js From 6998b964fe86539f3d16bc3d786bccac0c8922a8 Mon Sep 17 00:00:00 2001 From: extremeheat Date: Mon, 1 Apr 2024 09:12:15 +0000 Subject: [PATCH 10/50] force runs --- .github/helper-bot/index.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/helper-bot/index.js b/.github/helper-bot/index.js index 14a25781..7c0d06b1 100644 --- a/.github/helper-bot/index.js +++ b/.github/helper-bot/index.js @@ -117,6 +117,8 @@ async function fetchLatest () { if (issueStatus.isOpen) { await helper.updateIssue(issueStatus.id, issuePayload) + // TEMP TEST + core.setOutput('updateVersion', version) } else { const issue = await helper.createIssue(issuePayload) core.setOutput('updateVersion', version) From b08ebdd2c0cce0cc39958aaae3ab5f261b0a5de3 Mon Sep 17 00:00:00 2001 From: extremeheat Date: Mon, 1 Apr 2024 10:11:25 +0000 Subject: [PATCH 11/50] update helperbot workflow rabin --- .github/workflows/update-helper.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/update-helper.yml b/.github/workflows/update-helper.yml index 6b870ea3..5076f337 100644 --- a/.github/workflows/update-helper.yml +++ b/.github/workflows/update-helper.yml @@ -50,7 +50,9 @@ jobs: run: clang++-14 -g `llvm-config-14 --cxxflags --ldflags --libs` -o pdba.exe pdba.cpp working-directory: .github/helper-bot - name: Install radare2 - run: sudo apt-get install radare2 -y + # For some reason it's not appearing on GH Actions' apt.. so use snap + # run: sudo apt-get install radare2 -y + run: sudo snap install radare2 --classic - name: Install run: npm install gh-helpers # This step sets the "serverVersion" and "serverPath" outputs @@ -63,8 +65,7 @@ jobs: UPDATE_VERSION: ${{ needs.helper.outputs.updateVersion }} ISSUE_NUMBER: ${{ needs.helper.outputs.issueNumber }} - name: Dump .rodata strings - run: rabin2 -N 1 -z ${{ steps.update1Run.outputs.serverBin }} > .github/helper-bot/strings.txt - # radare2 -a x86 -b elf -qc 'izz~ro' ${{ steps.update1Run.outputs.serverBin }} > ./github/helper-bot/botstr.txt + run: /snap/bin/radare2.rabin2 -N 1 -z ${{ steps.update1Run.outputs.serverBin }} > .github/helper-bot/strings.txt - name: Dump .rodata run: readelf -x .rodata ${{ steps.update1Run.outputs.serverBin }} > .github/helper-bot/rodata.txt - name: Proc strings (Stage 0) From 909725cf33021a643eb893238cdb3e0ff6fa94bc Mon Sep 17 00:00:00 2001 From: extremeheat Date: Mon, 1 Apr 2024 10:16:59 +0000 Subject: [PATCH 12/50] update update1 --- .github/helper-bot/update1.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/helper-bot/update1.js b/.github/helper-bot/update1.js index 776dafe5..6701b626 100644 --- a/.github/helper-bot/update1.js +++ b/.github/helper-bot/update1.js @@ -2,6 +2,7 @@ const fs = require('fs') const core = require('@actions/core') const github = require('gh-helpers')() const bedrock = require('bedrock-protocol') +const { sleep } = require('bedrock-protocol/src/datatypes/util') const bedrockServer = require('minecraft-bedrock-server') const path = require('path') @@ -50,7 +51,8 @@ async function main (inputUpdateVer, inputIssueNo) { core.setOutput('serverPath', serverPath) core.setOutput('serverBin', serverPath + '/bedrock_server_symbols.debug') const handle = await bedrockServer.startServerAndWait(serverVersion, 60000, { root }) - const pong = await bedrock.ping({ host: 'localhost', port: 19132 }) + await sleep(2000) + const pong = await bedrock.ping({ host: 'localhost', port: 19130 }) updatedBody = updatedBody.replace('', `Protocol ID${pong.protocol} (${pong.version})`) try { await tryConnect({ protocolVersion: pong.protocol }) From f093ee0c8bb1cd2a12cb8e1908ba6f006ddbea4d Mon Sep 17 00:00:00 2001 From: extremeheat Date: Mon, 1 Apr 2024 10:20:51 +0000 Subject: [PATCH 13/50] wait longer --- .github/helper-bot/update1.js | 2 +- .github/workflows/update-helper.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/helper-bot/update1.js b/.github/helper-bot/update1.js index 6701b626..5ca139be 100644 --- a/.github/helper-bot/update1.js +++ b/.github/helper-bot/update1.js @@ -51,7 +51,7 @@ async function main (inputUpdateVer, inputIssueNo) { core.setOutput('serverPath', serverPath) core.setOutput('serverBin', serverPath + '/bedrock_server_symbols.debug') const handle = await bedrockServer.startServerAndWait(serverVersion, 60000, { root }) - await sleep(2000) + await sleep(9000) const pong = await bedrock.ping({ host: 'localhost', port: 19130 }) updatedBody = updatedBody.replace('', `Protocol ID${pong.protocol} (${pong.version})`) try { diff --git a/.github/workflows/update-helper.yml b/.github/workflows/update-helper.yml index 5076f337..4a0a5c7c 100644 --- a/.github/workflows/update-helper.yml +++ b/.github/workflows/update-helper.yml @@ -17,7 +17,7 @@ jobs: - name: Set up Node.js uses: actions/setup-node@master with: - node-version: 18.0.0 + node-version: 20.0.0 - name: Install deps in helper-bot run: npm install working-directory: .github/helper-bot @@ -42,7 +42,7 @@ jobs: - name: Set up Node.js uses: actions/setup-node@master with: - node-version: 18.0.0 + node-version: 20.0.0 - name: Compile disa run: clang++ -std=c++20 -O3 -o disa.exe disa.cpp working-directory: .github/helper-bot @@ -54,7 +54,7 @@ jobs: # run: sudo apt-get install radare2 -y run: sudo snap install radare2 --classic - name: Install - run: npm install gh-helpers + run: pnpm install gh-helpers # This step sets the "serverVersion" and "serverPath" outputs - name: Update Step 1 id: update1Run From 6465d4bc52a84b8c859fbe0c28503ee04eeaf9f0 Mon Sep 17 00:00:00 2001 From: extremeheat Date: Mon, 1 Apr 2024 10:23:03 +0000 Subject: [PATCH 14/50] drop pnpm --- .github/workflows/update-helper.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-helper.yml b/.github/workflows/update-helper.yml index 4a0a5c7c..e0bdce61 100644 --- a/.github/workflows/update-helper.yml +++ b/.github/workflows/update-helper.yml @@ -54,7 +54,7 @@ jobs: # run: sudo apt-get install radare2 -y run: sudo snap install radare2 --classic - name: Install - run: pnpm install gh-helpers + run: npm install gh-helpers # This step sets the "serverVersion" and "serverPath" outputs - name: Update Step 1 id: update1Run From 3cd720592a259ab5e8d4c7c97e3c1dcc682f1510 Mon Sep 17 00:00:00 2001 From: extremeheat Date: Mon, 1 Apr 2024 10:28:07 +0000 Subject: [PATCH 15/50] add ping timeout --- .github/helper-bot/update1.js | 5 +++-- src/createClient.js | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/helper-bot/update1.js b/.github/helper-bot/update1.js index 5ca139be..ac1a7a1e 100644 --- a/.github/helper-bot/update1.js +++ b/.github/helper-bot/update1.js @@ -51,8 +51,9 @@ async function main (inputUpdateVer, inputIssueNo) { core.setOutput('serverPath', serverPath) core.setOutput('serverBin', serverPath + '/bedrock_server_symbols.debug') const handle = await bedrockServer.startServerAndWait(serverVersion, 60000, { root }) - await sleep(9000) - const pong = await bedrock.ping({ host: 'localhost', port: 19130 }) + await sleep(1000 * 15) + console.log('Pinging...') + const pong = await bedrock.ping({ host: 'localhost', port: 19130, timeout: 2000 }) updatedBody = updatedBody.replace('', `Protocol ID${pong.protocol} (${pong.version})`) try { await tryConnect({ protocolVersion: pong.protocol }) diff --git a/src/createClient.js b/src/createClient.js index 154a8ae7..4318889b 100644 --- a/src/createClient.js +++ b/src/createClient.js @@ -83,10 +83,10 @@ function connect (client) { }) } -async function ping ({ host, port }) { +async function ping ({ host, port, timeout }) { const con = new RakClient({ host, port }) try { - return advertisement.fromServerName(await con.ping()) + return advertisement.fromServerName(await con.ping(timeout)) } finally { con.close() } From 13d57d05df843ca98dd60d04495ffa74d92abf4a Mon Sep 17 00:00:00 2001 From: extremeheat Date: Tue, 2 Apr 2024 07:08:03 -0400 Subject: [PATCH 16/50] helper-bot: updates --- .github/helper-bot/update1.js | 73 +++++++++++++++++++++-------- .github/helper-bot/update2.js | 3 +- .github/helper-bot/update3.js | 5 +- .github/workflows/update-helper.yml | 12 +++++ src/datatypes/util.js | 8 ++-- 5 files changed, 75 insertions(+), 26 deletions(-) diff --git a/.github/helper-bot/update1.js b/.github/helper-bot/update1.js index ac1a7a1e..75ff5b86 100644 --- a/.github/helper-bot/update1.js +++ b/.github/helper-bot/update1.js @@ -1,38 +1,62 @@ +/* eslint-disable no-var, no-extend-native */ const fs = require('fs') -const core = require('@actions/core') -const github = require('gh-helpers')() +const path = require('path') +const mcData = require('minecraft-data') +const latestSupportedProtocol = mcData.versions.bedrock[0].version const bedrock = require('bedrock-protocol') -const { sleep } = require('bedrock-protocol/src/datatypes/util') const bedrockServer = require('minecraft-bedrock-server') -const path = require('path') +let core +/** @type {import('gh-helpers').GithubHelper} */ +let github +if (process.env.CI) { + core = require('@actions/core') + github = require('gh-helpers')() +} else { + globalThis.isMocha = true + core = { setOutput: (name, value) => console.log(name, value) } + + github.findIssue = () => ({ body: '(Demo)' }) +} + +BigInt.prototype.toJSON = function () { + return this.toString() +} async function tryConnect (opts) { const client = bedrock.createClient({ host: 'localhost', - port: 19132, + port: 19130, username: 'test', offline: true }) client.on('connect_allowed', () => { Object.assign(client.options, opts) }) return new Promise((resolve, reject) => { + let timeout // eslint-disable-line + function done (data) { + client.close() + clearTimeout(timeout) + resolve(data) + } const collected = {} const forCollection = ['start_game', 'available_commands'] for (const packet of forCollection) { + console.log('Waiting for', packet) client.once(packet, (data) => { + console.log('Received', packet) collected[packet] = data fs.writeFileSync(path.join(__dirname, '/collected.json'), JSON.stringify(collected, null, 2)) + if (Object.keys(collected).length === forCollection.length) { + done(collected) + } }) - if (Object.keys(collected).length === forCollection.length) { - resolve(collected) - } } - setTimeout(() => { + timeout = setTimeout(() => { if (Object.keys(collected).length !== forCollection.length) { reject(Error('Unable to collect all packets')) } - }, 1000 * 60 * 6) + }, 1000 * 60 * 2) }) } @@ -44,16 +68,14 @@ async function main (inputUpdateVer, inputIssueNo) { if (serverVersion !== inputUpdateVer) { updatedBody = updatedBody.replace('', `Server version${serverVersion}`) } - const root = path.resolve(path.join(__dirname, '../../tools')) + const root = __dirname const serverPath = root + '/bds-' + serverVersion console.log('Server version', serverVersion, root, 'Server path', serverPath) core.setOutput('serverVersion', serverVersion) core.setOutput('serverPath', serverPath) core.setOutput('serverBin', serverPath + '/bedrock_server_symbols.debug') - const handle = await bedrockServer.startServerAndWait(serverVersion, 60000, { root }) - await sleep(1000 * 15) - console.log('Pinging...') - const pong = await bedrock.ping({ host: 'localhost', port: 19130, timeout: 2000 }) + const handle = await bedrockServer.startServerAndWait(serverVersion, 60000, { root: __dirname }) + const pong = await bedrock.ping({ host: '127.0.0.1', port: 19130 }) updatedBody = updatedBody.replace('', `Protocol ID${pong.protocol} (${pong.version})`) try { await tryConnect({ protocolVersion: pong.protocol }) @@ -62,20 +84,33 @@ async function main (inputUpdateVer, inputIssueNo) { console.error(e) updatedBody = updatedBody.replace('', 'Partly Already CompatibleNO') } - fs.writeFileSync(path.join(__dirname, '/updatedBody.md', updatedBody)) + fs.writeFileSync(path.join(__dirname, '/updatedBody.md'), updatedBody) await github.updateIssue({ number: inputIssueNo, body: updatedBody }) handle.kill() + + // Check if protocol version has changed + if (false && pong.protocol === latestSupportedProtocol) { + console.log('Protocol version has not changed') + // Close the github issue + await github.close(inputIssueNo, 'Protocol version has not changed, assuming no compatibility issues.') + core.setOutput('needsUpdate', false) + return + } else { + core.setOutput('needsUpdate', true) + core.setOutput('protocolVersion', pong.protocol) + } + console.log('✅ Finished working with Linux server binary') console.log('Working now on Windows') const winPath = serverPath.replace('bds-', 'bds-win-') - await bedrockServer.downloadServer(latestServers.windows.version3, winPath) + await bedrockServer.downloadServer(latestServers.windows.version3, { path: winPath }) core.setOutput('serverWinPath', winPath) core.setOutput('serverWinBin', winPath + '/bedrock_server.exe') core.setOutput('serverWinPdb', winPath + '/bedrock_server.pdb') console.log('✅ Finished working with Windows server binary') } -// main(process.env.UPDATE_VERSION, process.env.ISSUE_NUMBER) main(process.env.UPDATE_VERSION, process.env.ISSUE_NUMBER) +// main('1.20.73', 0) -// TODO: determine if protocol version has changed before doing anything \ No newline at end of file +// TODO: determine if protocol version has changed before doing anything diff --git a/.github/helper-bot/update2.js b/.github/helper-bot/update2.js index 22505b9e..c0b806d7 100644 --- a/.github/helper-bot/update2.js +++ b/.github/helper-bot/update2.js @@ -66,11 +66,10 @@ function writeStringsTSV (rodataDump, stringsFile) { const contents = subStr.split('ascii')[1]?.trim() if (!contents || !contents.match(/^[a-zA-Z0-9_:]+$/) || contents.length > 64) { if (subStr.includes('utf16le')) { - // sigh, some ascii strings are written as utf16le + // unfortunately some ascii strings are incorrectly written as utf16le const contents16 = subStr.split('utf16le')[1]?.trim() for (let j = 0; j < contents16.length; j += 1) { // write each char one by one as 1 length string - // const char = contents16[j] const newAddress = `0x${(parseInt(address, 16) + (j * 2)).toString(16).padStart(8, '0')}` result += `# fromU16LE ${address}\n` const ntString = readStringNTFromHexDump(newAddress) diff --git a/.github/helper-bot/update3.js b/.github/helper-bot/update3.js index 4bc4b6b4..767385b2 100644 --- a/.github/helper-bot/update3.js +++ b/.github/helper-bot/update3.js @@ -9,7 +9,8 @@ async function main () { } allStages.end() const artifact = await github.artifacts.createTextArtifact('updatorData', { - content: fs.readFileSync('merged.txt', 'latin1') + extracted: fs.readFileSync('merged.txt', 'latin1'), + collected: JSON.stringify(require('./collected.json')) }) console.log('Created artifact', artifact) const dispatch = await github.sendWorkflowDispatch({ @@ -21,6 +22,8 @@ async function main () { artifactId: artifact.id, artifactSize: artifact.size, updateVersion: process.env.UPDATE_VERSION, + serverVersion: process.env.SERVER_VERSION, + protocolVersion: process.env.PROTOCOL_VERSION, issueNo: process.env.ISSUE_NUMBER } }) diff --git a/.github/workflows/update-helper.yml b/.github/workflows/update-helper.yml index e0bdce61..0d9916e9 100644 --- a/.github/workflows/update-helper.yml +++ b/.github/workflows/update-helper.yml @@ -56,6 +56,7 @@ jobs: - name: Install run: npm install gh-helpers # This step sets the "serverVersion" and "serverPath" outputs + # Note, sometimes the serverVersionStr != clientVersionStr, so we handle that too via protocol check - name: Update Step 1 id: update1Run run: cd .github/helper-bot && node update1.js @@ -64,26 +65,37 @@ jobs: GITHUB_TOKEN: ${{ secrets.PAT_PASSWORD }} UPDATE_VERSION: ${{ needs.helper.outputs.updateVersion }} ISSUE_NUMBER: ${{ needs.helper.outputs.issueNumber }} + # The above will return a "needsUpdate" output. - name: Dump .rodata strings + if: steps.update1Run.outputs.needsUpdate run: /snap/bin/radare2.rabin2 -N 1 -z ${{ steps.update1Run.outputs.serverBin }} > .github/helper-bot/strings.txt - name: Dump .rodata + if: steps.update1Run.outputs.needsUpdate run: readelf -x .rodata ${{ steps.update1Run.outputs.serverBin }} > .github/helper-bot/rodata.txt - name: Proc strings (Stage 0) + if: steps.update1Run.outputs.needsUpdate run: node update2.js -s0 working-directory: .github/helper-bot - name: Process Linux server (Stage 1) + if: steps.update1Run.outputs.needsUpdate run: objdump -d --demangle --no-show-raw-insn --debugging ${{ steps.update1Run.outputs.serverBin }} | .github/helper-bot/disa.exe -s1 .github/helper-bot/strings.txt > .github/helper-bot/stage1.txt - name: Process Windows bin (Stage 2) + if: steps.update1Run.outputs.needsUpdate run: objdump -d --demangle -M intel ${{ steps.update1Run.outputs.serverWinBin }} | .github/helper-bot/disa.exe -s2 .github/helper-bot/stage1.txt > .github/helper-bot/stage2.txt - name: Resolve numbers (Stage 3) + if: steps.update1Run.outputs.needsUpdate run : node update2.js -s3 working-directory: .github/helper-bot - name: Resolve PDB symbols (Stage 4) + if: steps.update1Run.outputs.needsUpdate run: llvm-pdbutil dump --all ${{ steps.update1Run.outputs.serverWinPdb }} | .github/helper-bot/pdba.exe > .github/helper-bot/stage4.txt - name: Aggregate and finish + if: steps.update1Run.outputs.needsUpdate run: node update3.js working-directory: .github/helper-bot env: GITHUB_TOKEN: ${{ secrets.PAT_PASSWORD }} UPDATE_VERSION: ${{ needs.helper.outputs.updateVersion }} + SERVER_VERSION: ${{ steps.update1Run.outputs.serverVersion }} + PROTOCOL_VERSION: ${{ steps.update1Run.outputs.protocolVersion }} ISSUE_NUMBER: ${{ needs.helper.outputs.issueNumber }} \ No newline at end of file diff --git a/src/datatypes/util.js b/src/datatypes/util.js index 7070ce50..5c1c9261 100644 --- a/src/datatypes/util.js +++ b/src/datatypes/util.js @@ -22,10 +22,10 @@ function sleep (ms) { async function waitFor (cb, withTimeout, onTimeout) { let t - const ret = await Promise.race([ - new Promise((resolve, reject) => cb(resolve, reject)), - new Promise(resolve => { t = setTimeout(() => resolve('timeout'), withTimeout) }) - ]) + const ret = await new Promise((resolve) => { + t = setTimeout(() => resolve('timeout'), withTimeout) + cb(resolve) + }) clearTimeout(t) if (ret === 'timeout') await onTimeout() return ret From f369efaf07d9f99488e06609a957d471a778e659 Mon Sep 17 00:00:00 2001 From: extremeheat Date: Tue, 2 Apr 2024 07:14:17 -0400 Subject: [PATCH 17/50] helper-bot: update1 fix --- .github/helper-bot/update1.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/helper-bot/update1.js b/.github/helper-bot/update1.js index 75ff5b86..46b92c74 100644 --- a/.github/helper-bot/update1.js +++ b/.github/helper-bot/update1.js @@ -85,7 +85,7 @@ async function main (inputUpdateVer, inputIssueNo) { updatedBody = updatedBody.replace('', 'Partly Already CompatibleNO') } fs.writeFileSync(path.join(__dirname, '/updatedBody.md'), updatedBody) - await github.updateIssue({ number: inputIssueNo, body: updatedBody }) + await github.updateIssue(inputIssueNo, { body: updatedBody }) handle.kill() // Check if protocol version has changed @@ -112,5 +112,3 @@ async function main (inputUpdateVer, inputIssueNo) { main(process.env.UPDATE_VERSION, process.env.ISSUE_NUMBER) // main('1.20.73', 0) - -// TODO: determine if protocol version has changed before doing anything From cf0581755e81554d70a73ea0903b492c2e914bd1 Mon Sep 17 00:00:00 2001 From: extremeheat Date: Tue, 2 Apr 2024 07:25:56 -0400 Subject: [PATCH 18/50] add debug code --- .github/helper-bot/index.js | 1 + .github/workflows/update-helper.yml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.github/helper-bot/index.js b/.github/helper-bot/index.js index 7c0d06b1..6ba06138 100644 --- a/.github/helper-bot/index.js +++ b/.github/helper-bot/index.js @@ -119,6 +119,7 @@ async function fetchLatest () { await helper.updateIssue(issueStatus.id, issuePayload) // TEMP TEST core.setOutput('updateVersion', version) + core.setOutput('issueNumber', issueStatus.id) } else { const issue = await helper.createIssue(issuePayload) core.setOutput('updateVersion', version) diff --git a/.github/workflows/update-helper.yml b/.github/workflows/update-helper.yml index 0d9916e9..82baf6cc 100644 --- a/.github/workflows/update-helper.yml +++ b/.github/workflows/update-helper.yml @@ -37,6 +37,8 @@ jobs: # run if updateVersion exists if: ${{ needs.helper.outputs.updateVersion }} steps: + - name: Log debug + run: echo ${{ needs.helper.outputs.updateVersion }} ${{ needs.helper.outputs.issueNumber }} - name: Checkout repository uses: actions/checkout@master - name: Set up Node.js From 9b44a6bf7b112bce032d08703b4bd829e293e996 Mon Sep 17 00:00:00 2001 From: extremeheat Date: Tue, 2 Apr 2024 07:33:08 -0400 Subject: [PATCH 19/50] fix --- .github/helper-bot/update1.js | 2 +- .github/workflows/update-helper.yml | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/helper-bot/update1.js b/.github/helper-bot/update1.js index 46b92c74..2197c403 100644 --- a/.github/helper-bot/update1.js +++ b/.github/helper-bot/update1.js @@ -61,7 +61,7 @@ async function tryConnect (opts) { } async function main (inputUpdateVer, inputIssueNo) { - const issue = await github.findIssue({ number: inputIssueNo }) + const issue = await github.findIssue({ number: inputIssueNo, author: null }) const latestServers = await bedrockServer.getLatestVersions() let updatedBody = issue.body const serverVersion = latestServers.linux.version3 diff --git a/.github/workflows/update-helper.yml b/.github/workflows/update-helper.yml index 82baf6cc..0d9916e9 100644 --- a/.github/workflows/update-helper.yml +++ b/.github/workflows/update-helper.yml @@ -37,8 +37,6 @@ jobs: # run if updateVersion exists if: ${{ needs.helper.outputs.updateVersion }} steps: - - name: Log debug - run: echo ${{ needs.helper.outputs.updateVersion }} ${{ needs.helper.outputs.issueNumber }} - name: Checkout repository uses: actions/checkout@master - name: Set up Node.js From ace7344e0c5791cd6309bf387940e22b0a39cce7 Mon Sep 17 00:00:00 2001 From: extremeheat Date: Tue, 2 Apr 2024 07:46:14 -0400 Subject: [PATCH 20/50] fix issue upstream in gh-helpers --- .github/helper-bot/update1.js | 2 +- .github/workflows/update-helper.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/helper-bot/update1.js b/.github/helper-bot/update1.js index 2197c403..d09d8ee3 100644 --- a/.github/helper-bot/update1.js +++ b/.github/helper-bot/update1.js @@ -61,7 +61,7 @@ async function tryConnect (opts) { } async function main (inputUpdateVer, inputIssueNo) { - const issue = await github.findIssue({ number: inputIssueNo, author: null }) + const issue = await github.getIssue(inputIssueNo) const latestServers = await bedrockServer.getLatestVersions() let updatedBody = issue.body const serverVersion = latestServers.linux.version3 diff --git a/.github/workflows/update-helper.yml b/.github/workflows/update-helper.yml index 0d9916e9..4cdde313 100644 --- a/.github/workflows/update-helper.yml +++ b/.github/workflows/update-helper.yml @@ -54,7 +54,7 @@ jobs: # run: sudo apt-get install radare2 -y run: sudo snap install radare2 --classic - name: Install - run: npm install gh-helpers + run: npm install extremeheat/gh-helpers # This step sets the "serverVersion" and "serverPath" outputs # Note, sometimes the serverVersionStr != clientVersionStr, so we handle that too via protocol check - name: Update Step 1 From ef48e5c32ae357d4320059554397eb65338adb0e Mon Sep 17 00:00:00 2001 From: extremeheat Date: Tue, 2 Apr 2024 08:58:26 -0400 Subject: [PATCH 21/50] helper-bot updates --- .github/helper-bot/index.js | 17 +++++++++-------- .github/helper-bot/update1.js | 2 +- .github/helper-bot/update3.js | 7 +++++-- .github/workflows/update-helper.yml | 6 +++--- index.d.ts | 3 ++- src/options.js | 2 +- 6 files changed, 21 insertions(+), 16 deletions(-) diff --git a/.github/helper-bot/index.js b/.github/helper-bot/index.js index 6ba06138..766e8872 100644 --- a/.github/helper-bot/index.js +++ b/.github/helper-bot/index.js @@ -1,9 +1,10 @@ // Automatic version update checker for Minecraft bedrock edition. const fs = require('fs') +const { join } = require('path') const cp = require('child_process') const core = require('@actions/core') const helper = require('gh-helpers')() -const latestVesionEndpoint = 'https://itunes.apple.com/lookup?bundleId=com.mojang.minecraftpe&time=' + Date.now() +const latestVersionEndpoint = 'https://itunes.apple.com/lookup?bundleId=com.mojang.minecraftpe&time=' + Date.now() const changelogURL = 'https://feedback.minecraft.net/hc/en-us/sections/360001186971-Release-Changelogs' // Relevant infomation for us is: @@ -38,7 +39,6 @@ A new Minecraft Bedrock version is available (as of ${date}), version **${result ${commitData} ## Protocol Details -(I will close this issue automatically if "${result.version}" is added to index.d.ts on "master" and there are no X's below) @@ -46,6 +46,9 @@ ${commitData}
Name${result.version}
+*I'll try to close this issue automatically if the protocol version didn't change. +If the protocol version did change, the automatic update system will try to complete an update to minecraft-data and bedrock-protocol and if successful it will auto close this issue.* + ----- 🤖 I am a bot, I check for updates every 2 hours without a trigger. You can close this issue to prevent any further updates. @@ -78,15 +81,13 @@ function getCommitsInRepo (repo, containing, since) { } async function fetchLatest () { - if (!fs.existsSync('./results.json')) cp.execSync(`curl -L "${latestVesionEndpoint}" -o results.json`, { stdio: 'inherit', shell: true }) + cp.execSync(`curl -L "${latestVersionEndpoint}" -o results.json`, { stdio: 'inherit', shell: true }) const json = require('./results.json') const result = json.results[0] - // console.log(json) - if (!fs.existsSync('./index.d.ts')) cp.execSync('curl -LO https://raw.githubusercontent.com/PrismarineJS/bedrock-protocol/master/index.d.ts', { stdio: 'inherit', shell: true }) - const currentApi = fs.readFileSync('./index.d.ts', 'utf-8') - const supportedVersions = currentApi.match(/type Version = ([^\n]+)/)[1].replace(/\||'/g, ' ').split(' ').map(k => k.trim()).filter(k => k.length) - console.log(supportedVersions) + const currentTypes = fs.readFileSync(join(__dirname, '../../index.d.ts'), 'utf-8') + const supportedVersions = currentTypes.match(/type Version = ([^\n]+)/)[1].replace(/\||'/g, ' ').split(' ').map(k => k.trim()).filter(k => k.length) + console.log('Supported versions', supportedVersions) let { version, currentVersionReleaseDate, releaseNotes } = result console.log(version, currentVersionReleaseDate, releaseNotes) diff --git a/.github/helper-bot/update1.js b/.github/helper-bot/update1.js index d09d8ee3..62514772 100644 --- a/.github/helper-bot/update1.js +++ b/.github/helper-bot/update1.js @@ -103,7 +103,7 @@ async function main (inputUpdateVer, inputIssueNo) { console.log('✅ Finished working with Linux server binary') console.log('Working now on Windows') const winPath = serverPath.replace('bds-', 'bds-win-') - await bedrockServer.downloadServer(latestServers.windows.version3, { path: winPath }) + await bedrockServer.downloadServer(latestServers.windows.version3, { path: winPath, platform: 'windows' }) core.setOutput('serverWinPath', winPath) core.setOutput('serverWinBin', winPath + '/bedrock_server.exe') core.setOutput('serverWinPdb', winPath + '/bedrock_server.pdb') diff --git a/.github/helper-bot/update3.js b/.github/helper-bot/update3.js index 767385b2..ed35ff39 100644 --- a/.github/helper-bot/update3.js +++ b/.github/helper-bot/update3.js @@ -7,8 +7,11 @@ async function main () { for (const stage of stages) { allStages.write(fs.readFileSync(stage, 'latin1')) } - allStages.end() - const artifact = await github.artifacts.createTextArtifact('updatorData', { + allStages.end(upload) +} + +async function upload () { + const artifact = await github.artifacts.createTextArtifact('updateData-' + process.env.UPDATE_VERSION, { extracted: fs.readFileSync('merged.txt', 'latin1'), collected: JSON.stringify(require('./collected.json')) }) diff --git a/.github/workflows/update-helper.yml b/.github/workflows/update-helper.yml index 4cdde313..903ebf0a 100644 --- a/.github/workflows/update-helper.yml +++ b/.github/workflows/update-helper.yml @@ -53,8 +53,8 @@ jobs: # For some reason it's not appearing on GH Actions' apt.. so use snap # run: sudo apt-get install radare2 -y run: sudo snap install radare2 --classic - - name: Install - run: npm install extremeheat/gh-helpers + - name: Install bedrock-protocol + run: npm install gh-helpers # This step sets the "serverVersion" and "serverPath" outputs # Note, sometimes the serverVersionStr != clientVersionStr, so we handle that too via protocol check - name: Update Step 1 @@ -88,7 +88,7 @@ jobs: working-directory: .github/helper-bot - name: Resolve PDB symbols (Stage 4) if: steps.update1Run.outputs.needsUpdate - run: llvm-pdbutil dump --all ${{ steps.update1Run.outputs.serverWinPdb }} | .github/helper-bot/pdba.exe > .github/helper-bot/stage4.txt + run: llvm-pdbutil-14 dump --all ${{ steps.update1Run.outputs.serverWinPdb }} | .github/helper-bot/pdba.exe > .github/helper-bot/stage4.txt - name: Aggregate and finish if: steps.update1Run.outputs.needsUpdate run: node update3.js diff --git a/index.d.ts b/index.d.ts index c93b5a99..84a47a32 100644 --- a/index.d.ts +++ b/index.d.ts @@ -3,11 +3,12 @@ import { Realm } from 'prismarine-realms' import { ServerDeviceCodeResponse } from 'prismarine-auth' declare module 'bedrock-protocol' { + // Note: this tracks minecraft for iOS's version numbers, not server versions. type Version = '1.20.40' | '1.20.30' | '1.20.10' | '1.20.0' | '1.19.80' | '1.19.70' | '1.19.63' | '1.19.62' | '1.19.60' | '1.19.51' | '1.19.50' | '1.19.41' | '1.19.40' | '1.19.31' | '1.19.30' | '1.19.22' | '1.19.21' | '1.19.20' | '1.19.11' | '1.19.10' | '1.19.2' | '1.19.1' | '1.18.31' | '1.18.30' | '1.18.12' | '1.18.11' | '1.18.10' | '1.18.2' | '1.18.1' | '1.18.0' | '1.17.41' | '1.17.40' | '1.17.34' | '1.17.30' | '1.17.11' | '1.17.10' | '1.17.0' | '1.16.220' | '1.16.210' | '1.16.201' export interface Options { // The string version to start the client or server as - version?: Version + version?: string // For the client, the host of the server to connect to (default: 127.0.0.1) // For the server, the host to bind to (default: 0.0.0.0) host: string diff --git a/src/options.js b/src/options.js index ed4ff6cc..795f5957 100644 --- a/src/options.js +++ b/src/options.js @@ -2,7 +2,7 @@ const mcData = require('minecraft-data') // Minimum supported version (< will be kicked) const MIN_VERSION = '1.16.201' -// Currently supported verson. Note, clients with newer versions can still connect as long as data is in minecraft-data +// Currently supported version. Note, clients with newer versions can still connect as long as data is in minecraft-data const CURRENT_VERSION = '1.20.71' const Versions = Object.fromEntries(mcData.versions.bedrock.filter(e => e.releaseType === 'release').map(e => [e.minecraftVersion, e.version])) From 365ddc8b35e7545c10fa13b4fd2dd6cbebe9552c Mon Sep 17 00:00:00 2001 From: extremeheat Date: Tue, 2 Apr 2024 09:19:59 -0400 Subject: [PATCH 22/50] expose gh runtime to use artifacts --- .github/workflows/update-helper.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/update-helper.yml b/.github/workflows/update-helper.yml index 903ebf0a..61cf3b9e 100644 --- a/.github/workflows/update-helper.yml +++ b/.github/workflows/update-helper.yml @@ -89,6 +89,12 @@ jobs: - name: Resolve PDB symbols (Stage 4) if: steps.update1Run.outputs.needsUpdate run: llvm-pdbutil-14 dump --all ${{ steps.update1Run.outputs.serverWinPdb }} | .github/helper-bot/pdba.exe > .github/helper-bot/stage4.txt + # We use Artifacts API for cross repo com which is only avaliable within an Action, but not normal run jobs. + # So we need to store the keys in the env for update3 to use to do a workflow dispatch + # as in https://github.com/extremeheat/gh-helpers/blob/main/.github/workflows/ci.yml + - name: Expose GitHub Runtime + if: steps.update1Run.outputs.needsUpdate + uses: crazy-max/ghaction-github-runtime@v3 - name: Aggregate and finish if: steps.update1Run.outputs.needsUpdate run: node update3.js From faaf7f54e0776600af45be7d6bdf87bc2fefc848 Mon Sep 17 00:00:00 2001 From: extremeheat Date: Tue, 2 Apr 2024 14:24:00 +0000 Subject: [PATCH 23/50] fix wrong params --- .github/workflows/update-helper.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-helper.yml b/.github/workflows/update-helper.yml index 61cf3b9e..d582b7e5 100644 --- a/.github/workflows/update-helper.yml +++ b/.github/workflows/update-helper.yml @@ -78,7 +78,7 @@ jobs: working-directory: .github/helper-bot - name: Process Linux server (Stage 1) if: steps.update1Run.outputs.needsUpdate - run: objdump -d --demangle --no-show-raw-insn --debugging ${{ steps.update1Run.outputs.serverBin }} | .github/helper-bot/disa.exe -s1 .github/helper-bot/strings.txt > .github/helper-bot/stage1.txt + run: objdump -d --demangle --no-show-raw-insn ${{ steps.update1Run.outputs.serverBin }} | .github/helper-bot/disa.exe -s1 .github/helper-bot/strings.tsv > .github/helper-bot/stage1.txt - name: Process Windows bin (Stage 2) if: steps.update1Run.outputs.needsUpdate run: objdump -d --demangle -M intel ${{ steps.update1Run.outputs.serverWinBin }} | .github/helper-bot/disa.exe -s2 .github/helper-bot/stage1.txt > .github/helper-bot/stage2.txt From c51ab2434533324d3cdfa9118b141a53dffb9bb0 Mon Sep 17 00:00:00 2001 From: extremeheat Date: Thu, 4 Apr 2024 03:12:19 -0400 Subject: [PATCH 24/50] update c++ extractors --- .github/helper-bot/disa.cpp | 38 ++++++++++++++++++++++++++++++++++--- .github/helper-bot/pdba.cpp | 14 +++++++++++--- 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/.github/helper-bot/disa.cpp b/.github/helper-bot/disa.cpp index 787a1475..5cc0a853 100644 --- a/.github/helper-bot/disa.cpp +++ b/.github/helper-bot/disa.cpp @@ -77,6 +77,10 @@ bool hasSeenBlockClass(const std::string_view &blockClass) { return false; } +bool contains(const std::vector &vec, const std::string &str) { + return std::find(vec.begin(), vec.end(), str) != vec.end(); +} + void loadDisassembly(std::string filePath) { // *uses AT&T syntax ; too late to change now std::istream *disStream = &std::cin; @@ -105,6 +109,9 @@ void loadDisassembly(std::string filePath) { std::optional currentBlockData; + std::vector seenBlockIds; + // std::vector seenStates; + while (disStream->getline(buffer, bufferSize)) { // movabs $0x116b2c0, %rbx -> move the address to rbx if (buffer[10] == 'm' && buffer[11] == 'o' && buffer[12] == 'v' && buffer[13] == 'a' && buffer[14] == 'b' && @@ -117,12 +124,13 @@ void loadDisassembly(std::string filePath) { lastLoadedAddressAbsMovStr = addressStr; // auto addressInt = hexStr2Int(addressStr); - // if we are tracking a block, then print the constant + // B1. if we are tracking a block, then print the constant if (!trackingBlock.empty()) { // if line includes '#', then split by the comment and get the comment if (stringsMap.find(lastLoadedAddress) != stringsMap.end()) { std::cout << "BlockID\t" << trackingBlock << "\t" << addressStr << "\t" << stringsMap[lastLoadedAddress] << std::endl; + seenBlockIds.push_back(trackingBlock); } } } @@ -197,6 +205,7 @@ void loadDisassembly(std::string filePath) { size_t pos = line.find("VanillaBlockTypeIds::"); if (pos != std::string::npos) { trackingBlock = line.substr(pos + 21, line.size() - pos - 22); + // std::cout << "Now tracking: " << trackingBlock << "- " << line << std::endl; } if (currentBlockData.has_value()) { @@ -215,7 +224,22 @@ void loadDisassembly(std::string filePath) { } } } else { - trackingBlock.clear(); + // B1. cont. Sometimes the movabs with hash is not after 2x lea ops, so we dump what we have and continue + if (!trackingBlock.empty() && !(buffer[10] == 'm' && buffer[11] == 'o' && buffer[12] == 'v' && + buffer[13] == 'a' && buffer[14] == 'b' && buffer[15] == 's')) { + // If we've already seen the block, above check is not needed + if (!contains(seenBlockIds, trackingBlock)) { + // if line includes '#', then split by the comment and get the comment + if (stringsMap.find(lastLoadedAddress) != stringsMap.end()) { + std::cout << "BlockID\t" << trackingBlock << "\t" << "UNK" << "\t" << stringsMap[lastLoadedAddress] + << std::endl; + } + } + } + // lea/mov are both used to load constants before a call, so we can keep tracking if it's also a mov + if (!(buffer[10] == 'm' && buffer[11] == 'o' && buffer[12] == 'v')) { + trackingBlock.clear(); + } } // if a move over lea, we maybe loading block states @@ -247,6 +271,14 @@ void loadDisassembly(std::string filePath) { if (stringsMap.find(lastLoadedAddress) != stringsMap.end()) { std::cout << "VanillaState\t" << states << "\t" << lastLoadedAddressAbsMovStr << "\t" << stringsMap[lastLoadedAddress] << std::endl; + } else if (stringsMap.find(lastLastLoadedAddress) != stringsMap.end()) { + // try once more but with the last last loaded address... + // this can happen if another LEA code is between + std::cout << "VanillaState\t" << states << "\t" << lastLoadedAddressAbsMovStr << "\t" + << stringsMap[lastLastLoadedAddress] << std::endl; + } else { + // std::cout << "? NOT adding VanillaState\t" << states << " " << lastLoadedAddress << "\t" + // << lastLoadedAddressAbsMovStr << std::endl; } } } @@ -278,7 +310,7 @@ void loadDisassembly(std::string filePath) { auto pos = line.find("StateSerializationUtils::fromNBT<"); auto end = line.find(">", pos); auto substr = line.substr(pos + 33, end - pos - 33); - std::cout << "StateSerializer\t" << substr << std::endl; + // std::cout << "StateSerializer\t" << substr << std::endl; inStateSerializer = std::string(substr); } else { inStateSerializer.clear(); diff --git a/.github/helper-bot/pdba.cpp b/.github/helper-bot/pdba.cpp index d844e9c5..4eb32648 100644 --- a/.github/helper-bot/pdba.cpp +++ b/.github/helper-bot/pdba.cpp @@ -20,11 +20,19 @@ void loadDump() { auto vanillaStateName = demangled.substr(vanillaStatesPos + 15); if (demangled.find("Variant") != std::string::npos) { - std::cout << "VST " << "\t" << vanillaStateName << "\t" << "int" << std::endl; + std::cout << "VST\t" << vanillaStateName << "\t" << "int" << std::endl; } else if (demangled.find("Variant") != std::string::npos) { - std::cout << "VST " << "\t" << vanillaStateName << "\t" << "bool" << std::endl; + std::cout << "VST\t" << vanillaStateName << "\t" << "bool" << std::endl; } else { - std::cout << "VST " << "\t" << vanillaStateName << "\t" << "string" << std::endl; + // Capture what's in the Variant<...> + auto variantPos = demangled.find("Variant<"); + if (variantPos != std::string::npos) { + auto variantEndPos = demangled.find(">", variantPos); + auto variantType = demangled.substr(variantPos + 8, variantEndPos - variantPos - 8); + std::cout << "VST\t" << vanillaStateName << "\t" << "string" << "\t" << variantType << std::endl; + } else { + std::cout << "VST\t" << vanillaStateName << "\t" << "string" << std::endl; + } } } } From ba3c5e5dd92a148dac74c33c286b7e083264f015 Mon Sep 17 00:00:00 2001 From: extremeheat Date: Thu, 4 Apr 2024 04:12:25 -0400 Subject: [PATCH 25/50] update --- .github/helper-bot/disa.cpp | 1 + .github/helper-bot/pdba.cpp | 61 +++++++++++++++++++++++-------------- 2 files changed, 39 insertions(+), 23 deletions(-) diff --git a/.github/helper-bot/disa.cpp b/.github/helper-bot/disa.cpp index 5cc0a853..f6140d63 100644 --- a/.github/helper-bot/disa.cpp +++ b/.github/helper-bot/disa.cpp @@ -5,6 +5,7 @@ #include #include #include +#include void ZeroMemory(char *buffer, int size) { for (int i = 0; i < size; i++) { diff --git a/.github/helper-bot/pdba.cpp b/.github/helper-bot/pdba.cpp index 4eb32648..6c493395 100644 --- a/.github/helper-bot/pdba.cpp +++ b/.github/helper-bot/pdba.cpp @@ -2,6 +2,40 @@ #include #include +void readVanillaState(std::string &demangled) { + auto vanillaStatesPos = demangled.find("VanillaStates::"); + if (vanillaStatesPos == std::string::npos) { + return; + } + auto vanillaStateName = demangled.substr(vanillaStatesPos + 15); + if (demangled.find("Variant") != std::string::npos) { + std::cout << "VST\t" << vanillaStateName << "\t" << "int" << std::endl; + } else if (demangled.find("Variant") != std::string::npos) { + std::cout << "VST\t" << vanillaStateName << "\t" << "bool" << std::endl; + } else { + // Capture what's in the Variant<...> + auto variantPos = demangled.find("Variant<"); + if (variantPos != std::string::npos) { + auto variantEndPos = demangled.find(">", variantPos); + auto variantType = demangled.substr(variantPos + 8, variantEndPos - variantPos - 8); + std::cout << "VST\t" << vanillaStateName << "\t" << "string" << "\t" << variantType << std::endl; + } else { + std::cout << "VST\t" << vanillaStateName << "\t" << "string" << std::endl; + } + } +} + +void readConstant(std::string &demangled) { + // enum CodeBuilder::ProtocolVersion const SharedConstants::CodeBuilderProtocolVersion + auto sharedPos = demangled.find("SharedConstants::"); + if (sharedPos == std::string::npos) { + return; + } + auto sharedName = demangled.substr(sharedPos + 17); + auto type = demangled.substr(0, sharedPos - 1); + std::cout << "SCT\t" << sharedName << "\t" << type << std::endl; +} + void loadDump() { std::string line; while (std::getline(std::cin, line)) { @@ -10,29 +44,10 @@ void loadDump() { std::string mangledName = line.substr(pos + 1, line.size() - pos - 2); std::string demangled = llvm::demangle(mangledName); - if (mangledName.find("VanillaStates") == std::string::npos) { - continue; - } - auto vanillaStatesPos = demangled.find("VanillaStates::"); - if (vanillaStatesPos == std::string::npos) { - continue; - } - - auto vanillaStateName = demangled.substr(vanillaStatesPos + 15); - if (demangled.find("Variant") != std::string::npos) { - std::cout << "VST\t" << vanillaStateName << "\t" << "int" << std::endl; - } else if (demangled.find("Variant") != std::string::npos) { - std::cout << "VST\t" << vanillaStateName << "\t" << "bool" << std::endl; - } else { - // Capture what's in the Variant<...> - auto variantPos = demangled.find("Variant<"); - if (variantPos != std::string::npos) { - auto variantEndPos = demangled.find(">", variantPos); - auto variantType = demangled.substr(variantPos + 8, variantEndPos - variantPos - 8); - std::cout << "VST\t" << vanillaStateName << "\t" << "string" << "\t" << variantType << std::endl; - } else { - std::cout << "VST\t" << vanillaStateName << "\t" << "string" << std::endl; - } + if (mangledName.find("VanillaStates") != std::string::npos) { + readVanillaState(demangled); + } else if (mangledName.find("SharedConstants") != std::string::npos) { + readConstant(demangled); } } } From d43d08dc855f1bb660670812e268b9e122da0329 Mon Sep 17 00:00:00 2001 From: extremeheat Date: Thu, 4 Apr 2024 05:43:10 -0400 Subject: [PATCH 26/50] update rodata handling, workflow dispatching --- .github/helper-bot/.gitignore | 2 +- .github/helper-bot/disa.cpp | 11 +++++++++++ .github/helper-bot/download.js | 28 ---------------------------- .github/helper-bot/update2.js | 26 ++++++-------------------- .github/helper-bot/update3.js | 20 ++++++++++++-------- .github/workflows/update-helper.yml | 2 +- 6 files changed, 31 insertions(+), 58 deletions(-) delete mode 100644 .github/helper-bot/download.js diff --git a/.github/helper-bot/.gitignore b/.github/helper-bot/.gitignore index 7d9c40f6..4c8b7d5d 100644 --- a/.github/helper-bot/.gitignore +++ b/.github/helper-bot/.gitignore @@ -1,5 +1,5 @@ bds-* *.exe stage* -rodata.txt +rodata.* strings.* \ No newline at end of file diff --git a/.github/helper-bot/disa.cpp b/.github/helper-bot/disa.cpp index f6140d63..6cc64221 100644 --- a/.github/helper-bot/disa.cpp +++ b/.github/helper-bot/disa.cpp @@ -111,6 +111,7 @@ void loadDisassembly(std::string filePath) { std::optional currentBlockData; std::vector seenBlockIds; + std::vector seenConstants; // std::vector seenStates; while (disStream->getline(buffer, bufferSize)) { @@ -201,6 +202,16 @@ void loadDisassembly(std::string filePath) { stateEntries[inStateSerializer].push_back(stringsMap[addressInt]); } } + + size_t constPos = line.find("SharedConstants::"); + if (constPos != std::string::npos) { + auto sharedName = line.substr(constPos + 17, line.size() - constPos - 18); + auto sharedNameStr = std::string(sharedName); + if (!contains(seenConstants, sharedNameStr)) { + seenConstants.push_back(sharedNameStr); + std::cout << "Const\t" << sharedName << "\t" << addressStr << std::endl; + } + } } size_t pos = line.find("VanillaBlockTypeIds::"); diff --git a/.github/helper-bot/download.js b/.github/helper-bot/download.js deleted file mode 100644 index b77449ac..00000000 --- a/.github/helper-bot/download.js +++ /dev/null @@ -1,28 +0,0 @@ -const htmlURL = 'https://www.minecraft.net/en-us/download/server/bedrock' - -async function getLatestVersions () { - const html = await fetch(htmlURL).then(res => res.text()) - // Find ' Download ' - const links = [...html.matchAll(/a href="(.*?)" /g)].map(match => match[1]) - - function forOS (os) { - const url = links.find(link => link.includes(os + '/')) - if (!url) return null - const version4 = url.match(/bedrock-server-(\d+\.\d+\.\d+\.\d+)\.zip/)[1] - const version3 = version4.split('.').slice(0, 3).join('.') - return { version4, version3, url } - } - - return { - linux: forOS('linux'), - windows: forOS('win'), - macos: forOS('osx'), - preview: { - linux: forOS('linux-preview'), - windows: forOS('win-preview'), - macos: forOS('osx-preview') - } - } -} - -getLatestVersions().then(console.log).catch(console.error) diff --git a/.github/helper-bot/update2.js b/.github/helper-bot/update2.js index c0b806d7..31949b2f 100644 --- a/.github/helper-bot/update2.js +++ b/.github/helper-bot/update2.js @@ -2,25 +2,11 @@ const { join } = require('path') const fs = require('fs') -/* -Hex dump of section '.rodata': - 0x00ba0080 00004a42 000000c0 17b7d138 9a99193e ..JB.......8...> - 0x00ba0090 00008043 000020c2 c3f5e8be 0000003c ...C.. ........< -*/ - +// 00ba0080 function loadRoDataHexDump (filePath) { - const rodataHexDump = fs.readFileSync(filePath, 'latin1') - const hexDumpLines = rodataHexDump.split('\n') - const buffer = Buffer.alloc(hexDumpLines.length * 16) - let initialBufferIx - let bufferIx = 0 - for (const line of hexDumpLines) { - if (!line.startsWith(' ')) continue - initialBufferIx ||= line.slice(2, 12) - const data = line.slice(13, 48).replaceAll(' ', '') - bufferIx += buffer.write(data, bufferIx, 'hex') - } - const initialBufferOffset = parseInt(initialBufferIx, 16) + const rodataHexDump = fs.readFileSync(filePath) + const buffer = rodataHexDump + const initialBufferOffset = Buffer.from(rodataHexDump.slice(-8).toString('latin1'), 'latin1').readUInt32BE(0) function readStringNTFromHexDump (addr) { const addrNum = typeof addr === 'string' ? parseInt(addr, 16) : addr @@ -114,12 +100,12 @@ function postProc (rodataDump, s1file) { const stage = process.argv[2] if (stage === '-s0') { writeStringsTSV( - join(__dirname, 'rodata.txt'), + join(__dirname, 'rodata.bin'), join(__dirname, 'strings.txt') ) } else if (stage === '-s3') { postProc( - join(__dirname, 'rodata.txt'), + join(__dirname, 'rodata.bin'), join(__dirname, 'stage1.txt') ) } diff --git a/.github/helper-bot/update3.js b/.github/helper-bot/update3.js index ed35ff39..42c8bfa4 100644 --- a/.github/helper-bot/update3.js +++ b/.github/helper-bot/update3.js @@ -13,7 +13,8 @@ async function main () { async function upload () { const artifact = await github.artifacts.createTextArtifact('updateData-' + process.env.UPDATE_VERSION, { extracted: fs.readFileSync('merged.txt', 'latin1'), - collected: JSON.stringify(require('./collected.json')) + collected: JSON.stringify(require('./collected.json')), + rodata: fs.readFileSync('rodata.bin') }) console.log('Created artifact', artifact) const dispatch = await github.sendWorkflowDispatch({ @@ -21,13 +22,16 @@ async function upload () { workflow: 'dispatch.yml', branch: 'main', inputs: { - repoData: await github.getRepoDetails(), - artifactId: artifact.id, - artifactSize: artifact.size, - updateVersion: process.env.UPDATE_VERSION, - serverVersion: process.env.SERVER_VERSION, - protocolVersion: process.env.PROTOCOL_VERSION, - issueNo: process.env.ISSUE_NUMBER + action: 'minecraft/bedrockDataUpdate', + payload: JSON.stringify({ + repoData: await github.getRepoDetails(), + artifactId: artifact.id, + artifactSize: artifact.size, + updateVersion: process.env.UPDATE_VERSION, + serverVersion: process.env.SERVER_VERSION, + protocolVersion: process.env.PROTOCOL_VERSION, + issueNo: process.env.ISSUE_NUMBER + }) } }) console.log('Sent workflow dispatch', dispatch) diff --git a/.github/workflows/update-helper.yml b/.github/workflows/update-helper.yml index d582b7e5..59a8358e 100644 --- a/.github/workflows/update-helper.yml +++ b/.github/workflows/update-helper.yml @@ -71,7 +71,7 @@ jobs: run: /snap/bin/radare2.rabin2 -N 1 -z ${{ steps.update1Run.outputs.serverBin }} > .github/helper-bot/strings.txt - name: Dump .rodata if: steps.update1Run.outputs.needsUpdate - run: readelf -x .rodata ${{ steps.update1Run.outputs.serverBin }} > .github/helper-bot/rodata.txt + run: objcopy -O binary -j .rodata ${{ steps.update1Run.outputs.serverBin }} .github/helper-bot/rodata.bin - name: Proc strings (Stage 0) if: steps.update1Run.outputs.needsUpdate run: node update2.js -s0 From 923f79fc1e9aac9503c379e8ed5f5707fdd99c10 Mon Sep 17 00:00:00 2001 From: extremeheat Date: Thu, 4 Apr 2024 10:29:34 +0000 Subject: [PATCH 27/50] dump snap in favor of updating package registry --- .github/workflows/update-helper.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/update-helper.yml b/.github/workflows/update-helper.yml index 59a8358e..d7622dbe 100644 --- a/.github/workflows/update-helper.yml +++ b/.github/workflows/update-helper.yml @@ -49,10 +49,14 @@ jobs: - name: Compile pdba run: clang++-14 -g `llvm-config-14 --cxxflags --ldflags --libs` -o pdba.exe pdba.cpp working-directory: .github/helper-bot + # For some reason radare2 isn't installable on GH with apt as it's missing + # the `http://archive.ubuntu.com/ubuntu focal/universe amd64` repo. So we can either + # install with snap, or try to update the apt repos. + - name: Update system packages + run: sudo sed -i '1i deb http://archive.ubuntu.com/ubuntu focal/universe amd64' /etc/apt/sources.list - name: Install radare2 - # For some reason it's not appearing on GH Actions' apt.. so use snap - # run: sudo apt-get install radare2 -y - run: sudo snap install radare2 --classic + run: sudo apt update && sudo apt-get install radare2 -y + # run: sudo snap install radare2 --classic - name: Install bedrock-protocol run: npm install gh-helpers # This step sets the "serverVersion" and "serverPath" outputs From be6e3dc16db9750640f4246848715b1934639bb2 Mon Sep 17 00:00:00 2001 From: extremeheat Date: Thu, 4 Apr 2024 10:30:47 +0000 Subject: [PATCH 28/50] update workflows --- .github/workflows/update-helper.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/update-helper.yml b/.github/workflows/update-helper.yml index d7622dbe..1189adae 100644 --- a/.github/workflows/update-helper.yml +++ b/.github/workflows/update-helper.yml @@ -72,7 +72,8 @@ jobs: # The above will return a "needsUpdate" output. - name: Dump .rodata strings if: steps.update1Run.outputs.needsUpdate - run: /snap/bin/radare2.rabin2 -N 1 -z ${{ steps.update1Run.outputs.serverBin }} > .github/helper-bot/strings.txt + # run: /snap/bin/radare2.rabin2 -N 1 -z ${{ steps.update1Run.outputs.serverBin }} > .github/helper-bot/strings.txt + run: rabin2 -N 1 -z ${{ steps.update1Run.outputs.serverBin }} > .github/helper-bot/strings.txt - name: Dump .rodata if: steps.update1Run.outputs.needsUpdate run: objcopy -O binary -j .rodata ${{ steps.update1Run.outputs.serverBin }} .github/helper-bot/rodata.bin From 5a9755ef91a6dd4eb4dbafd884fd3caae94924bb Mon Sep 17 00:00:00 2001 From: extremeheat Date: Thu, 4 Apr 2024 10:51:03 +0000 Subject: [PATCH 29/50] update helper --- .github/workflows/update-helper.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/update-helper.yml b/.github/workflows/update-helper.yml index 1189adae..2982d466 100644 --- a/.github/workflows/update-helper.yml +++ b/.github/workflows/update-helper.yml @@ -49,11 +49,10 @@ jobs: - name: Compile pdba run: clang++-14 -g `llvm-config-14 --cxxflags --ldflags --libs` -o pdba.exe pdba.cpp working-directory: .github/helper-bot - # For some reason radare2 isn't installable on GH with apt as it's missing - # the `http://archive.ubuntu.com/ubuntu focal/universe amd64` repo. So we can either - # install with snap, or try to update the apt repos. + # For some reason radare2 isn't installable on GH with apt as it's missing the + # ubuntu/archive/focal/universe repo. So we can either install with snap, or try to update the apt repos. - name: Update system packages - run: sudo sed -i '1i deb http://archive.ubuntu.com/ubuntu focal/universe amd64' /etc/apt/sources.list + run: sudo add-apt-repository "deb http://archive.ubuntu.com/ubuntu focal universe" - name: Install radare2 run: sudo apt update && sudo apt-get install radare2 -y # run: sudo snap install radare2 --classic @@ -72,7 +71,6 @@ jobs: # The above will return a "needsUpdate" output. - name: Dump .rodata strings if: steps.update1Run.outputs.needsUpdate - # run: /snap/bin/radare2.rabin2 -N 1 -z ${{ steps.update1Run.outputs.serverBin }} > .github/helper-bot/strings.txt run: rabin2 -N 1 -z ${{ steps.update1Run.outputs.serverBin }} > .github/helper-bot/strings.txt - name: Dump .rodata if: steps.update1Run.outputs.needsUpdate From f4ec61e13bcbd8217ed4e348f0059dc6b4622775 Mon Sep 17 00:00:00 2001 From: extremeheat Date: Thu, 4 Apr 2024 12:46:12 +0000 Subject: [PATCH 30/50] revert back to using snap --- .github/workflows/update-helper.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/update-helper.yml b/.github/workflows/update-helper.yml index 2982d466..ccb068b0 100644 --- a/.github/workflows/update-helper.yml +++ b/.github/workflows/update-helper.yml @@ -51,11 +51,12 @@ jobs: working-directory: .github/helper-bot # For some reason radare2 isn't installable on GH with apt as it's missing the # ubuntu/archive/focal/universe repo. So we can either install with snap, or try to update the apt repos. - - name: Update system packages - run: sudo add-apt-repository "deb http://archive.ubuntu.com/ubuntu focal universe" + # We get newer version with snap so let's use that... + # - name: Update system packages + # run: sudo add-apt-repository "deb http://archive.ubuntu.com/ubuntu focal universe" - name: Install radare2 - run: sudo apt update && sudo apt-get install radare2 -y - # run: sudo snap install radare2 --classic + # run: sudo apt update && sudo apt-get install radare2 -y + run: sudo snap install radare2 --classic - name: Install bedrock-protocol run: npm install gh-helpers # This step sets the "serverVersion" and "serverPath" outputs @@ -71,7 +72,7 @@ jobs: # The above will return a "needsUpdate" output. - name: Dump .rodata strings if: steps.update1Run.outputs.needsUpdate - run: rabin2 -N 1 -z ${{ steps.update1Run.outputs.serverBin }} > .github/helper-bot/strings.txt + run: /snap/bin/radare2.rabin2 -N 1 -z ${{ steps.update1Run.outputs.serverBin }} > .github/helper-bot/strings.txt - name: Dump .rodata if: steps.update1Run.outputs.needsUpdate run: objcopy -O binary -j .rodata ${{ steps.update1Run.outputs.serverBin }} .github/helper-bot/rodata.bin From f9ac75de7de862f1b92a6494d633b9402e451f62 Mon Sep 17 00:00:00 2001 From: extremeheat Date: Thu, 4 Apr 2024 13:08:51 +0000 Subject: [PATCH 31/50] some debugging --- .github/helper-bot/update3.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/helper-bot/update3.js b/.github/helper-bot/update3.js index 42c8bfa4..094f1ab4 100644 --- a/.github/helper-bot/update3.js +++ b/.github/helper-bot/update3.js @@ -14,7 +14,8 @@ async function upload () { const artifact = await github.artifacts.createTextArtifact('updateData-' + process.env.UPDATE_VERSION, { extracted: fs.readFileSync('merged.txt', 'latin1'), collected: JSON.stringify(require('./collected.json')), - rodata: fs.readFileSync('rodata.bin') + rodata: fs.readFileSync('rodata.bin'), + strings: fs.readFileSync('strings.tsv') }) console.log('Created artifact', artifact) const dispatch = await github.sendWorkflowDispatch({ From 611577c8d1ca907d0b7c9c735c74dc35eb779730 Mon Sep 17 00:00:00 2001 From: extremeheat Date: Thu, 4 Apr 2024 11:20:50 -0400 Subject: [PATCH 32/50] Drop dep on rabin2 Read rodata directly in disa1 --- .github/helper-bot/disa.cpp | 92 ++++++++++++++++++----------- .github/helper-bot/update2.js | 2 +- .github/workflows/update-helper.yml | 17 +----- 3 files changed, 58 insertions(+), 53 deletions(-) diff --git a/.github/helper-bot/disa.cpp b/.github/helper-bot/disa.cpp index 6cc64221..fab5d797 100644 --- a/.github/helper-bot/disa.cpp +++ b/.github/helper-bot/disa.cpp @@ -38,27 +38,47 @@ unsigned int hexStr2IntLE(const std::string_view &hexStr) { return swapped; } -std::map stringsMap; -void loadStrings(std::string filePath) { - // Load the strings file TSV into a map - std::ifstream stringsStream(filePath, std::ios::binary); - if (!stringsStream.is_open()) { - std::cerr << "Failed to open file: " << filePath << std::endl; +char *roData; +int roDataOffset; +int roDataEnd; +void loadRoData(std::string binFile) { + std::ifstream binStream(binFile, std::ios::binary); + if (!binStream.is_open()) { + std::cerr << "Failed to open file: " << binFile << std::endl; return; } - const int bufferSize = 1024 * 64; - char buffer[bufferSize]; - while (stringsStream.getline(buffer, bufferSize)) { - std::string_view line(buffer); - size_t pos = line.find('\t'); - if (pos != std::string::npos) { - std::string_view key = line.substr(0, pos); - std::string_view value = line.substr(pos + 1, line.size() - pos - 1); - unsigned int keyInt = hexStr2Int(key); - stringsMap[keyInt] = std::string(value); - } + binStream.seekg(0, std::ios::end); + int size = binStream.tellg(); + binStream.seekg(0, std::ios::beg); + roData = new char[size]; + binStream.read(roData, size); + binStream.close(); + // End 9 bytes holds offset of the rodata section + std::string_view offsetStr(roData + size - 9, 8); + roDataOffset = hexStr2Int(offsetStr); + roDataEnd = roDataOffset + size; +} + +bool isAddressInRoData(unsigned int address) { return address >= roDataOffset && address < roDataEnd; } +bool isRoDataStringNT(unsigned int address) { + if (!isAddressInRoData(address)) { + return false; + } + auto bufferOffset = address - roDataOffset; + return roData[bufferOffset] != '\0'; +} + +// Get a null-terminated string from the rodata section +std::string getRoDataStringNT(unsigned int offset) { + if (!isRoDataStringNT(offset)) { + return ""; + } + auto bufferOffset = offset - roDataOffset; + int len = 0; + while (roData[bufferOffset + len] != '\0') { + len++; } - stringsStream.close(); + return std::string(roData + bufferOffset, len); } struct CurrentBlockData { @@ -129,9 +149,9 @@ void loadDisassembly(std::string filePath) { // B1. if we are tracking a block, then print the constant if (!trackingBlock.empty()) { // if line includes '#', then split by the comment and get the comment - if (stringsMap.find(lastLoadedAddress) != stringsMap.end()) { - std::cout << "BlockID\t" << trackingBlock << "\t" << addressStr << "\t" << stringsMap[lastLoadedAddress] - << std::endl; + if (isRoDataStringNT(lastLoadedAddress)) { + std::string str = getRoDataStringNT(lastLoadedAddress); + std::cout << "BlockID\t" << trackingBlock << "\t" << addressStr << "\t" << str << std::endl; seenBlockIds.push_back(trackingBlock); } } @@ -198,8 +218,9 @@ void loadDisassembly(std::string filePath) { if (inStateSerializer.size() > 0) { // we are interested in capturing all loaded constants inside the state serializer - if (stringsMap.find(addressInt) != stringsMap.end()) { - stateEntries[inStateSerializer].push_back(stringsMap[addressInt]); + if (isRoDataStringNT(addressInt)) { + auto str = getRoDataStringNT(addressInt); + stateEntries[inStateSerializer].push_back(str); } } @@ -242,9 +263,9 @@ void loadDisassembly(std::string filePath) { // If we've already seen the block, above check is not needed if (!contains(seenBlockIds, trackingBlock)) { // if line includes '#', then split by the comment and get the comment - if (stringsMap.find(lastLoadedAddress) != stringsMap.end()) { - std::cout << "BlockID\t" << trackingBlock << "\t" << "UNK" << "\t" << stringsMap[lastLoadedAddress] - << std::endl; + if (isRoDataStringNT(lastLoadedAddress)) { + auto str = getRoDataStringNT(lastLoadedAddress); + std::cout << "BlockID\t" << trackingBlock << "\t" << "UNK" << "\t" << str << std::endl; } } } @@ -280,14 +301,12 @@ void loadDisassembly(std::string filePath) { continue; } auto states = comment.substr(statesPos + 15, comment.size() - statesPos - 16); - if (stringsMap.find(lastLoadedAddress) != stringsMap.end()) { - std::cout << "VanillaState\t" << states << "\t" << lastLoadedAddressAbsMovStr << "\t" - << stringsMap[lastLoadedAddress] << std::endl; - } else if (stringsMap.find(lastLastLoadedAddress) != stringsMap.end()) { - // try once more but with the last last loaded address... - // this can happen if another LEA code is between - std::cout << "VanillaState\t" << states << "\t" << lastLoadedAddressAbsMovStr << "\t" - << stringsMap[lastLastLoadedAddress] << std::endl; + if (isRoDataStringNT(lastLoadedAddress)) { + auto str = getRoDataStringNT(lastLoadedAddress); + std::cout << "VanillaState\t" << states << "\t" << lastLoadedAddressAbsMovStr << "\t" << str << std::endl; + } else if (isRoDataStringNT(lastLastLoadedAddress)) { + auto str = getRoDataStringNT(lastLastLoadedAddress); + std::cout << "VanillaState\t" << states << "\t" << lastLoadedAddressAbsMovStr << "\t" << str << std::endl; } else { // std::cout << "? NOT adding VanillaState\t" << states << " " << lastLoadedAddress << "\t" // << lastLoadedAddressAbsMovStr << std::endl; @@ -475,7 +494,7 @@ void loadDisassembly2(std::string filePath) { int main(int argc, char **argv) { if (argc < 3) { - std::cerr << "Usage: disa -s1 [dis]" << std::endl; + std::cerr << "Usage: disa -s1 [dis]" << std::endl; std::cerr << "Usage: disa -s2 [dis]" << std::endl; return 1; } @@ -490,11 +509,12 @@ int main(int argc, char **argv) { std::cout << "(waiting for stdin)" << std::endl; } if (stage == "-s1") { - loadStrings(file); + loadRoData(file); loadDisassembly(disFile); } else if (stage == "-s2") { loadStage1(file); loadDisassembly2(disFile); } + printf("Done\n"); return 0; } diff --git a/.github/helper-bot/update2.js b/.github/helper-bot/update2.js index 31949b2f..6947d136 100644 --- a/.github/helper-bot/update2.js +++ b/.github/helper-bot/update2.js @@ -6,7 +6,7 @@ const fs = require('fs') function loadRoDataHexDump (filePath) { const rodataHexDump = fs.readFileSync(filePath) const buffer = rodataHexDump - const initialBufferOffset = Buffer.from(rodataHexDump.slice(-8).toString('latin1'), 'latin1').readUInt32BE(0) + const initialBufferOffset = Buffer.from(rodataHexDump.slice(-9, -1).toString('latin1'), 'latin1').readUInt32BE(0) function readStringNTFromHexDump (addr) { const addrNum = typeof addr === 'string' ? parseInt(addr, 16) : addr diff --git a/.github/workflows/update-helper.yml b/.github/workflows/update-helper.yml index ccb068b0..5cf36f6f 100644 --- a/.github/workflows/update-helper.yml +++ b/.github/workflows/update-helper.yml @@ -49,14 +49,6 @@ jobs: - name: Compile pdba run: clang++-14 -g `llvm-config-14 --cxxflags --ldflags --libs` -o pdba.exe pdba.cpp working-directory: .github/helper-bot - # For some reason radare2 isn't installable on GH with apt as it's missing the - # ubuntu/archive/focal/universe repo. So we can either install with snap, or try to update the apt repos. - # We get newer version with snap so let's use that... - # - name: Update system packages - # run: sudo add-apt-repository "deb http://archive.ubuntu.com/ubuntu focal universe" - - name: Install radare2 - # run: sudo apt update && sudo apt-get install radare2 -y - run: sudo snap install radare2 --classic - name: Install bedrock-protocol run: npm install gh-helpers # This step sets the "serverVersion" and "serverPath" outputs @@ -70,19 +62,12 @@ jobs: UPDATE_VERSION: ${{ needs.helper.outputs.updateVersion }} ISSUE_NUMBER: ${{ needs.helper.outputs.issueNumber }} # The above will return a "needsUpdate" output. - - name: Dump .rodata strings - if: steps.update1Run.outputs.needsUpdate - run: /snap/bin/radare2.rabin2 -N 1 -z ${{ steps.update1Run.outputs.serverBin }} > .github/helper-bot/strings.txt - name: Dump .rodata if: steps.update1Run.outputs.needsUpdate run: objcopy -O binary -j .rodata ${{ steps.update1Run.outputs.serverBin }} .github/helper-bot/rodata.bin - - name: Proc strings (Stage 0) - if: steps.update1Run.outputs.needsUpdate - run: node update2.js -s0 - working-directory: .github/helper-bot - name: Process Linux server (Stage 1) if: steps.update1Run.outputs.needsUpdate - run: objdump -d --demangle --no-show-raw-insn ${{ steps.update1Run.outputs.serverBin }} | .github/helper-bot/disa.exe -s1 .github/helper-bot/strings.tsv > .github/helper-bot/stage1.txt + run: objdump -d --demangle --no-show-raw-insn ${{ steps.update1Run.outputs.serverBin }} | .github/helper-bot/disa.exe -s1 .github/helper-bot/rodata.bin > .github/helper-bot/stage1.txt - name: Process Windows bin (Stage 2) if: steps.update1Run.outputs.needsUpdate run: objdump -d --demangle -M intel ${{ steps.update1Run.outputs.serverWinBin }} | .github/helper-bot/disa.exe -s2 .github/helper-bot/stage1.txt > .github/helper-bot/stage2.txt From 6df441fc0cbb0185325e322048bc76d40e805765 Mon Sep 17 00:00:00 2001 From: extremeheat Date: Thu, 4 Apr 2024 11:25:14 -0400 Subject: [PATCH 33/50] update update3 --- .github/helper-bot/update3.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/helper-bot/update3.js b/.github/helper-bot/update3.js index 094f1ab4..522f7b58 100644 --- a/.github/helper-bot/update3.js +++ b/.github/helper-bot/update3.js @@ -13,9 +13,7 @@ async function main () { async function upload () { const artifact = await github.artifacts.createTextArtifact('updateData-' + process.env.UPDATE_VERSION, { extracted: fs.readFileSync('merged.txt', 'latin1'), - collected: JSON.stringify(require('./collected.json')), - rodata: fs.readFileSync('rodata.bin'), - strings: fs.readFileSync('strings.tsv') + collected: JSON.stringify(require('./collected.json')) }) console.log('Created artifact', artifact) const dispatch = await github.sendWorkflowDispatch({ From 4af14d941902ac099810053f5c70c651f0aa3dc8 Mon Sep 17 00:00:00 2001 From: extremeheat Date: Thu, 4 Apr 2024 12:26:41 -0400 Subject: [PATCH 34/50] add debug --- .github/workflows/update-helper.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/update-helper.yml b/.github/workflows/update-helper.yml index 5cf36f6f..934f34f5 100644 --- a/.github/workflows/update-helper.yml +++ b/.github/workflows/update-helper.yml @@ -65,6 +65,10 @@ jobs: - name: Dump .rodata if: steps.update1Run.outputs.needsUpdate run: objcopy -O binary -j .rodata ${{ steps.update1Run.outputs.serverBin }} .github/helper-bot/rodata.bin + # Debug + - run: objdump -v + - run: objdump -d --demangle --no-show-raw-insn ${{ steps.update1Run.outputs.serverBin }} > bedrock.asm + - run: sha1sum bedrock.asm - name: Process Linux server (Stage 1) if: steps.update1Run.outputs.needsUpdate run: objdump -d --demangle --no-show-raw-insn ${{ steps.update1Run.outputs.serverBin }} | .github/helper-bot/disa.exe -s1 .github/helper-bot/rodata.bin > .github/helper-bot/stage1.txt From 70caca71cc2e3e4b5c4426af956cb150ace1bf76 Mon Sep 17 00:00:00 2001 From: extremeheat Date: Fri, 5 Apr 2024 02:41:26 -0400 Subject: [PATCH 35/50] update att instr handling --- .github/helper-bot/disa.cpp | 305 ++++++++++++++++++++++------ .github/workflows/update-helper.yml | 4 +- 2 files changed, 239 insertions(+), 70 deletions(-) diff --git a/.github/helper-bot/disa.cpp b/.github/helper-bot/disa.cpp index fab5d797..8cc7f98e 100644 --- a/.github/helper-bot/disa.cpp +++ b/.github/helper-bot/disa.cpp @@ -81,6 +81,194 @@ std::string getRoDataStringNT(unsigned int offset) { return std::string(roData + bufferOffset, len); } +// +// 708c23b: lea 0x1647ede(%rip),%rsi # 86d4120 +enum InstructionType { NO_INSTR, MOVABS, MOV, LEA, CALL, OTHER, FUNCTION_START }; +struct Instruction { + InstructionType type; + bool isFunctionStart; + char *asciiAddressStart; + char *asciiAddressEnd; + char *asciiOpStart; + char *asciiOpEnd; + char *asciiOperandsStart; + char *asciiOperandsEnd; + char *asciiCommentStart; + char *asciiCommentEnd; + unsigned int commentAddr; + char *commentSymbolStart; + char *commentSymbolEnd; +}; +void clearInstruction(Instruction &instr) { + instr.type = NO_INSTR; + instr.isFunctionStart = false; + instr.asciiAddressStart = nullptr; + instr.asciiAddressEnd = nullptr; + instr.asciiOpStart = nullptr; + instr.asciiOpEnd = nullptr; + instr.asciiOperandsStart = nullptr; + instr.asciiOperandsEnd = nullptr; + instr.asciiCommentStart = nullptr; + instr.asciiCommentEnd = nullptr; +} +void parseAttLine(char *buffer, Instruction &instr) { + instr.asciiAddressStart = buffer; + if (buffer[0] == ' ') { + instr.isFunctionStart = false; + } else { + instr.isFunctionStart = true; + } + + if (instr.isFunctionStart) { + // 0000000002c44530 : + bool readingAddress = true; + bool readingSymbol = false; + int i = 0; + for (;; i++) { + auto c = buffer[i]; + if (c == '\0') + break; + if (c == ' ' || c == '\t') { + if (readingAddress) { + readingAddress = false; + readingSymbol = true; + instr.asciiAddressEnd = buffer + i; + instr.asciiOpStart = buffer + i + 1; + } + } + } + instr.asciiOpEnd = buffer + i; + // op end here holds the symbol name + // remove the colon + instr.asciiOpEnd--; + } else { + bool readingAddress = true; + bool readingOp = false; + bool readingOperands = false; + bool readingComment = false; + for (int i = 0; true; i++) { + auto c = buffer[i]; + if (c == '\0') + break; + + if (readingAddress) { + for (int j = i; true; j++) { + auto c = buffer[j]; + if (c == '\0') + break; + if (c == ':') { + readingAddress = false; + readingOp = true; + instr.asciiAddressEnd = buffer + j; + i = j; + break; + } + } + } else if (readingOp) { + for (int j = i; true; j++) { + auto c = buffer[j]; + if (c == '\0') + break; + if (c == ' ' || c == '\t') { + if (instr.asciiOpStart) { + readingOp = false; + readingOperands = true; + instr.asciiOpEnd = buffer + j; + i = j; + break; + } + } else if (!instr.asciiOpStart) { + instr.asciiOpStart = buffer + j; + } + } + } else if (readingOperands) { + for (int j = i; true; j++) { + auto c = buffer[j]; + if (c == '#' || c == '\0') { + readingOperands = false; + readingComment = true; + instr.asciiOperandsEnd = buffer + j; + i = j; + break; + } else if (!(c == ' ' || c == '\t')) { + if (!instr.asciiOperandsStart) { + instr.asciiOperandsStart = buffer + j; + } + } + } + } else if (readingComment) { + for (int j = i; true; j++) { + auto c = buffer[j]; + if (c == '\0') { + readingComment = false; + instr.asciiCommentEnd = buffer + j; + i = j; + break; + } else if (!instr.asciiCommentStart) { + instr.asciiCommentStart = buffer + j; + } + } + } + } + } + + // Sanity check: make sure we have at least an op start and end (this also covers functions) + if (!instr.asciiOpStart || !instr.asciiOpEnd) { + instr.type = NO_INSTR; + instr.isFunctionStart = false; + return; + } + + if (instr.isFunctionStart) { + instr.type = FUNCTION_START; + return; + } + + auto op = instr.asciiOpStart; + if (op[0] == 'm' && op[1] == 'o' && op[2] == 'v' && op[3] == 'a' && op[4] == 'b' && op[5] == 's') { + instr.type = MOVABS; + } else if (op[0] == 'm' && op[1] == 'o' && op[2] == 'v') { + instr.type = MOV; + } else if (op[0] == 'l' && op[1] == 'e' && op[2] == 'a') { + instr.type = LEA; + } else if (op[0] == 'c' && op[1] == 'a' && op[2] == 'l' && op[3] == 'l') { + instr.type = CALL; + } else { + instr.type = OTHER; + } + + if (instr.asciiCommentStart && instr.asciiCommentEnd) { + // Comment Start: [ 86d4120 ] + // Comment End: [] + // iterate until the first '<' and then until the first '>' + char *asciiCommentAddrStart = instr.asciiCommentStart + 1; + char *asciiCommentAddrEnd = nullptr; + char *asciiCommentSymbolStart = nullptr; + char *asciiCommentSymbolEnd = nullptr; + for (int i = 0; true; i++) { + auto c = asciiCommentAddrStart[i]; + if (c == '\0') + break; + if (c == '<' && !asciiCommentAddrEnd) { + asciiCommentAddrEnd = asciiCommentAddrStart + i; + asciiCommentSymbolStart = asciiCommentAddrStart + i + 1; + } else if (c == '>') { + asciiCommentSymbolEnd = asciiCommentAddrStart + i; + break; + } + } + if (asciiCommentAddrEnd && asciiCommentSymbolStart && asciiCommentSymbolEnd) { + instr.commentAddr = + hexStr2Int(std::string_view(asciiCommentAddrStart, asciiCommentAddrEnd - asciiCommentAddrStart)); + instr.commentSymbolStart = asciiCommentSymbolStart; + instr.commentSymbolEnd = asciiCommentSymbolEnd; + } + } +} +// + +// void parseIntelLine() {} + struct CurrentBlockData { std::string blockName; std::string blockClass; @@ -134,44 +322,46 @@ void loadDisassembly(std::string filePath) { std::vector seenConstants; // std::vector seenStates; + Instruction instr; + while (disStream->getline(buffer, bufferSize)) { + parseAttLine(buffer, instr); + if (instr.type == NO_INSTR) { + goto finish; + } + // movabs $0x116b2c0, %rbx -> move the address to rbx - if (buffer[10] == 'm' && buffer[11] == 'o' && buffer[12] == 'v' && buffer[13] == 'a' && buffer[14] == 'b' && - buffer[15] == 's') { + if (instr.type == MOVABS) { std::string_view line(buffer); size_t pos = line.find("$"); size_t endPos = line.find(","); if (pos != std::string::npos && endPos != std::string::npos) { - auto addressStr = line.substr(pos + 1, endPos - pos - 1); - lastLoadedAddressAbsMovStr = addressStr; - // auto addressInt = hexStr2Int(addressStr); + auto loadedStr = line.substr(pos + 1, endPos - pos - 1); + lastLoadedAddressAbsMovStr = loadedStr; // B1. if we are tracking a block, then print the constant if (!trackingBlock.empty()) { // if line includes '#', then split by the comment and get the comment if (isRoDataStringNT(lastLoadedAddress)) { std::string str = getRoDataStringNT(lastLoadedAddress); - std::cout << "BlockID\t" << trackingBlock << "\t" << addressStr << "\t" << str << std::endl; + std::cout << "BlockID\t" << trackingBlock << "\t" << loadedStr << "\t" << str << std::endl; seenBlockIds.push_back(trackingBlock); } } } } - // callq 745ef90 (HashedString const&, int&&)> - if (buffer[10] == 'c' && buffer[11] == 'a' && buffer[12] == 'l' && buffer[13] == 'l') { + if (instr.type == CALL) { std::string_view line(buffer); - - // if line contains registerBlock, then it's a block registration - size_t registerPos = line.find("registerBlock<"); - if (registerPos != std::string::npos) { + size_t pos = line.find("registerBlock<"); + if (pos != std::string::npos) { if (currentBlockData.has_value()) { blockData.push_back(currentBlockData.value()); currentBlockData.reset(); } // class name is between "registerBlock<" and "," - size_t classStart = registerPos; + size_t classStart = pos; size_t classEnd = line.find(",", classStart); auto blockClass = line.substr(classStart + 14, classEnd - classStart - 14); if (trackingBlock.empty()) { @@ -180,7 +370,6 @@ void loadDisassembly(std::string filePath) { } } else { currentBlockData = CurrentBlockData{.blockClass = std::string(blockClass)}; - // std::cout << "BlockRegistration\t" << trackingBlock << "\t" << blockClass << std::endl; currentBlockData->blockName = trackingBlock; } } @@ -200,45 +389,35 @@ void loadDisassembly(std::string filePath) { } } - if (buffer[10] == 'l' && buffer[11] == 'e' && buffer[12] == 'a') { + if (instr.type == LEA) { std::string_view line(buffer); - size_t pos = line.find("#"); - if (pos != std::string::npos) { - auto comment = line.substr(pos + 2, line.size() - pos - 1); - // above bounds to skip the comment+space - // split the comment by the first space - auto addressPos = comment.find(" "); - if (addressPos != std::string::npos) { - auto addressStr = comment.substr(0, addressPos); - auto addressInt = hexStr2Int(addressStr); - - lastLastLoadedAddress = lastLoadedAddress; - lastLoadedAddress = addressInt; + if (instr.commentSymbolStart && instr.commentSymbolEnd) { + // there is a "# 86d4858 " comment + lastLastLoadedAddress = lastLoadedAddress; + lastLoadedAddress = instr.commentAddr; - if (inStateSerializer.size() > 0) { - // we are interested in capturing all loaded constants inside the state serializer - if (isRoDataStringNT(addressInt)) { - auto str = getRoDataStringNT(addressInt); - stateEntries[inStateSerializer].push_back(str); - } + if (inStateSerializer.size() > 0) { + // we are interested in capturing all loaded constants inside the state serializer + if (isRoDataStringNT(instr.commentAddr)) { + auto str = getRoDataStringNT(instr.commentAddr); + stateEntries[inStateSerializer].push_back(str); } + } - size_t constPos = line.find("SharedConstants::"); - if (constPos != std::string::npos) { - auto sharedName = line.substr(constPos + 17, line.size() - constPos - 18); - auto sharedNameStr = std::string(sharedName); - if (!contains(seenConstants, sharedNameStr)) { - seenConstants.push_back(sharedNameStr); - std::cout << "Const\t" << sharedName << "\t" << addressStr << std::endl; - } + size_t constPos = line.find("SharedConstants::"); + if (constPos != std::string::npos) { + auto sharedName = line.substr(constPos + 17, line.size() - constPos - 18); + auto sharedNameStr = std::string(sharedName); + if (!contains(seenConstants, sharedNameStr)) { + seenConstants.push_back(sharedNameStr); + std::cout << "Const\t" << sharedName << "\t" << instr.commentAddr << std::endl; } } size_t pos = line.find("VanillaBlockTypeIds::"); if (pos != std::string::npos) { trackingBlock = line.substr(pos + 21, line.size() - pos - 22); - // std::cout << "Now tracking: " << trackingBlock << "- " << line << std::endl; } if (currentBlockData.has_value()) { @@ -246,10 +425,10 @@ void loadDisassembly(std::string filePath) { size_t statesPos = line.find("VanillaStates::"); if (statesPos != std::string::npos) { // ensure there's no + in the symbol - if (comment.find("+") != std::string::npos) { + if (line.find("+") != std::string::npos) { continue; } - auto end = comment.find(">"); + auto end = line.find(">"); auto states = line.substr(statesPos + 15, line.size() - statesPos - 16); auto statesStr = std::string(states); currentBlockData->stateKeys.push_back(statesStr); @@ -258,8 +437,7 @@ void loadDisassembly(std::string filePath) { } } else { // B1. cont. Sometimes the movabs with hash is not after 2x lea ops, so we dump what we have and continue - if (!trackingBlock.empty() && !(buffer[10] == 'm' && buffer[11] == 'o' && buffer[12] == 'v' && - buffer[13] == 'a' && buffer[14] == 'b' && buffer[15] == 's')) { + if (!trackingBlock.empty() && instr.type != MOVABS) { // If we've already seen the block, above check is not needed if (!contains(seenBlockIds, trackingBlock)) { // if line includes '#', then split by the comment and get the comment @@ -270,37 +448,28 @@ void loadDisassembly(std::string filePath) { } } // lea/mov are both used to load constants before a call, so we can keep tracking if it's also a mov - if (!(buffer[10] == 'm' && buffer[11] == 'o' && buffer[12] == 'v')) { + if (instr.type != MOV) { trackingBlock.clear(); } } // if a move over lea, we maybe loading block states - if (buffer[10] == 'm' && buffer[11] == 'o' && buffer[12] == 'v') { + if (instr.type == MOV) { std::string_view line(buffer); // if line includes '#', then split by the comment and get the comment - size_t pos = line.find("#"); - if (pos != std::string::npos) { - auto comment = line.substr(pos + 2, line.size() - pos - 1); - - // Some reason some blocks are loaded outside of the block registry + if (instr.commentAddr) { if (isInBlockRegistry || currentBlockData.has_value()) { - auto addressPos = comment.find(" "); - if (addressPos != std::string::npos) { - auto addressStr = comment.substr(0, addressPos); - auto addressInt = hexStr2Int(addressStr); - lastLastLoadedAddress = lastLoadedAddress; - lastLoadedAddress = addressInt; - } + lastLastLoadedAddress = lastLoadedAddress; + lastLoadedAddress = instr.commentAddr; } else if (isInGlobalBlock) { // State Registration? - auto statesPos = comment.find("VanillaStates::"); + size_t statesPos = line.find("VanillaStates::"); if (statesPos != std::string::npos) { // ensure there's no + in the symbol - if (comment.find("+") != std::string::npos) { - continue; + if (line.find("+") != std::string::npos) { + goto finish; } - auto states = comment.substr(statesPos + 15, comment.size() - statesPos - 16); + auto states = line.substr(statesPos + 15, line.size() - statesPos - 16); if (isRoDataStringNT(lastLoadedAddress)) { auto str = getRoDataStringNT(lastLoadedAddress); std::cout << "VanillaState\t" << states << "\t" << lastLoadedAddressAbsMovStr << "\t" << str << std::endl; @@ -317,11 +486,11 @@ void loadDisassembly(std::string filePath) { } // if buffer ends with a colon, then it's a new block - if (buffer[0] == '0') { + if (instr.isFunctionStart) { std::string_view line(buffer); trackingBlock.clear(); // globals initialization - if (line.find("<_GLOBAL_") != std::string::npos) { + if (line.find("_GLOBAL_") != std::string::npos) { isInGlobalBlock = true; } else { isInGlobalBlock = false; @@ -341,13 +510,14 @@ void loadDisassembly(std::string filePath) { auto pos = line.find("StateSerializationUtils::fromNBT<"); auto end = line.find(">", pos); auto substr = line.substr(pos + 33, end - pos - 33); - // std::cout << "StateSerializer\t" << substr << std::endl; inStateSerializer = std::string(substr); } else { inStateSerializer.clear(); } } + finish: ZeroMemory(buffer, bufferSize); + clearInstruction(instr); } if (!filePath.empty()) { @@ -516,5 +686,6 @@ int main(int argc, char **argv) { loadDisassembly2(disFile); } printf("Done\n"); + std::cerr << "Done" << std::endl; return 0; } diff --git a/.github/workflows/update-helper.yml b/.github/workflows/update-helper.yml index 934f34f5..564d9a34 100644 --- a/.github/workflows/update-helper.yml +++ b/.github/workflows/update-helper.yml @@ -65,10 +65,8 @@ jobs: - name: Dump .rodata if: steps.update1Run.outputs.needsUpdate run: objcopy -O binary -j .rodata ${{ steps.update1Run.outputs.serverBin }} .github/helper-bot/rodata.bin - # Debug - - run: objdump -v - run: objdump -d --demangle --no-show-raw-insn ${{ steps.update1Run.outputs.serverBin }} > bedrock.asm - - run: sha1sum bedrock.asm + - run: head -n 20 bedrock.asm - name: Process Linux server (Stage 1) if: steps.update1Run.outputs.needsUpdate run: objdump -d --demangle --no-show-raw-insn ${{ steps.update1Run.outputs.serverBin }} | .github/helper-bot/disa.exe -s1 .github/helper-bot/rodata.bin > .github/helper-bot/stage1.txt From e5ad0e085c5c8461e9051861f856b0c72beaa88e Mon Sep 17 00:00:00 2001 From: extremeheat Date: Fri, 5 Apr 2024 02:54:47 -0400 Subject: [PATCH 36/50] helperbot update --- .github/helper-bot/update1.js | 4 +++- .github/workflows/update-helper.yml | 7 ++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/helper-bot/update1.js b/.github/helper-bot/update1.js index 62514772..f23327bd 100644 --- a/.github/helper-bot/update1.js +++ b/.github/helper-bot/update1.js @@ -75,7 +75,8 @@ async function main (inputUpdateVer, inputIssueNo) { core.setOutput('serverPath', serverPath) core.setOutput('serverBin', serverPath + '/bedrock_server_symbols.debug') const handle = await bedrockServer.startServerAndWait(serverVersion, 60000, { root: __dirname }) - const pong = await bedrock.ping({ host: '127.0.0.1', port: 19130 }) + await new Promise((resolve) => setTimeout(resolve, 2000)) + const pong = await bedrock.ping({ host: '127.0.0.1', port: 19130, timeout: 4000 }) updatedBody = updatedBody.replace('', `Protocol ID${pong.protocol} (${pong.version})`) try { await tryConnect({ protocolVersion: pong.protocol }) @@ -86,6 +87,7 @@ async function main (inputUpdateVer, inputIssueNo) { } fs.writeFileSync(path.join(__dirname, '/updatedBody.md'), updatedBody) await github.updateIssue(inputIssueNo, { body: updatedBody }) + console.log('Updated issue body', inputIssueNo, updatedBody) handle.kill() // Check if protocol version has changed diff --git a/.github/workflows/update-helper.yml b/.github/workflows/update-helper.yml index 564d9a34..36eced47 100644 --- a/.github/workflows/update-helper.yml +++ b/.github/workflows/update-helper.yml @@ -65,11 +65,12 @@ jobs: - name: Dump .rodata if: steps.update1Run.outputs.needsUpdate run: objcopy -O binary -j .rodata ${{ steps.update1Run.outputs.serverBin }} .github/helper-bot/rodata.bin - - run: objdump -d --demangle --no-show-raw-insn ${{ steps.update1Run.outputs.serverBin }} > bedrock.asm - - run: head -n 20 bedrock.asm + - run: sha1sum ${{ steps.update1Run.outputs.serverBin }} + - run: llvm-objdump-14 -d --demangle --no-show-raw-insn ${{ steps.update1Run.outputs.serverBin }} > llvm.asm + - run: head -n 20 llvm.asm - name: Process Linux server (Stage 1) if: steps.update1Run.outputs.needsUpdate - run: objdump -d --demangle --no-show-raw-insn ${{ steps.update1Run.outputs.serverBin }} | .github/helper-bot/disa.exe -s1 .github/helper-bot/rodata.bin > .github/helper-bot/stage1.txt + run: llvm-objdump-14 -d --demangle --no-show-raw-insn ${{ steps.update1Run.outputs.serverBin }} | .github/helper-bot/disa.exe -s1 .github/helper-bot/rodata.bin > .github/helper-bot/stage1.txt - name: Process Windows bin (Stage 2) if: steps.update1Run.outputs.needsUpdate run: objdump -d --demangle -M intel ${{ steps.update1Run.outputs.serverWinBin }} | .github/helper-bot/disa.exe -s2 .github/helper-bot/stage1.txt > .github/helper-bot/stage2.txt From ad6fb8ca22295a826eab473c3b0f45daf45f9b6c Mon Sep 17 00:00:00 2001 From: extremeheat Date: Fri, 5 Apr 2024 03:23:44 -0400 Subject: [PATCH 37/50] disa: fix clearInstruction --- .github/helper-bot/disa.cpp | 4 ++++ .github/helper-bot/update1.js | 1 + .github/workflows/update-helper.yml | 1 + 3 files changed, 6 insertions(+) diff --git a/.github/helper-bot/disa.cpp b/.github/helper-bot/disa.cpp index 8cc7f98e..f7d4c27c 100644 --- a/.github/helper-bot/disa.cpp +++ b/.github/helper-bot/disa.cpp @@ -57,6 +57,7 @@ void loadRoData(std::string binFile) { std::string_view offsetStr(roData + size - 9, 8); roDataOffset = hexStr2Int(offsetStr); roDataEnd = roDataOffset + size; + fprintf(stderr, "Opened rodata file '%s', size: %d, offset: %d\n", binFile.c_str(), size, roDataOffset); } bool isAddressInRoData(unsigned int address) { return address >= roDataOffset && address < roDataEnd; } @@ -110,6 +111,9 @@ void clearInstruction(Instruction &instr) { instr.asciiOperandsEnd = nullptr; instr.asciiCommentStart = nullptr; instr.asciiCommentEnd = nullptr; + instr.commentAddr = 0; + instr.commentSymbolStart = nullptr; + instr.commentSymbolEnd = nullptr; } void parseAttLine(char *buffer, Instruction &instr) { instr.asciiAddressStart = buffer; diff --git a/.github/helper-bot/update1.js b/.github/helper-bot/update1.js index f23327bd..267e8038 100644 --- a/.github/helper-bot/update1.js +++ b/.github/helper-bot/update1.js @@ -63,6 +63,7 @@ async function tryConnect (opts) { async function main (inputUpdateVer, inputIssueNo) { const issue = await github.getIssue(inputIssueNo) const latestServers = await bedrockServer.getLatestVersions() + console.log('Issue data', issue) let updatedBody = issue.body const serverVersion = latestServers.linux.version3 if (serverVersion !== inputUpdateVer) { diff --git a/.github/workflows/update-helper.yml b/.github/workflows/update-helper.yml index 36eced47..0d9923eb 100644 --- a/.github/workflows/update-helper.yml +++ b/.github/workflows/update-helper.yml @@ -66,6 +66,7 @@ jobs: if: steps.update1Run.outputs.needsUpdate run: objcopy -O binary -j .rodata ${{ steps.update1Run.outputs.serverBin }} .github/helper-bot/rodata.bin - run: sha1sum ${{ steps.update1Run.outputs.serverBin }} + - run: sha1sum .github/helper-bot/rodata.bin - run: llvm-objdump-14 -d --demangle --no-show-raw-insn ${{ steps.update1Run.outputs.serverBin }} > llvm.asm - run: head -n 20 llvm.asm - name: Process Linux server (Stage 1) From d753cc3cd330ca724380ddebd4a78eec68afbd54 Mon Sep 17 00:00:00 2001 From: extremeheat Date: Fri, 5 Apr 2024 03:36:42 -0400 Subject: [PATCH 38/50] workflow fix --- .github/helper-bot/update1.js | 2 +- .github/workflows/update-helper.yml | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/helper-bot/update1.js b/.github/helper-bot/update1.js index 267e8038..8efcaff7 100644 --- a/.github/helper-bot/update1.js +++ b/.github/helper-bot/update1.js @@ -81,7 +81,7 @@ async function main (inputUpdateVer, inputIssueNo) { updatedBody = updatedBody.replace('', `Protocol ID${pong.protocol} (${pong.version})`) try { await tryConnect({ protocolVersion: pong.protocol }) - updatedBody = updatedBody.replace('', '') + updatedBody = updatedBody.replace('', 'Partly Already CompatibleYes') } catch (e) { console.error(e) updatedBody = updatedBody.replace('', 'Partly Already CompatibleNO') diff --git a/.github/workflows/update-helper.yml b/.github/workflows/update-helper.yml index 0d9923eb..5b5e2854 100644 --- a/.github/workflows/update-helper.yml +++ b/.github/workflows/update-helper.yml @@ -64,14 +64,16 @@ jobs: # The above will return a "needsUpdate" output. - name: Dump .rodata if: steps.update1Run.outputs.needsUpdate - run: objcopy -O binary -j .rodata ${{ steps.update1Run.outputs.serverBin }} .github/helper-bot/rodata.bin + run: | + objcopy -O binary -j .rodata ${{ steps.update1Run.outputs.serverBin }} .github/helper-bot/rodata.bin + readelf -S ${{ steps.update1Run.outputs.serverBin }} | grep .rodata >> .github/helper-bot/rodata.bin - run: sha1sum ${{ steps.update1Run.outputs.serverBin }} - run: sha1sum .github/helper-bot/rodata.bin - run: llvm-objdump-14 -d --demangle --no-show-raw-insn ${{ steps.update1Run.outputs.serverBin }} > llvm.asm - run: head -n 20 llvm.asm - name: Process Linux server (Stage 1) if: steps.update1Run.outputs.needsUpdate - run: llvm-objdump-14 -d --demangle --no-show-raw-insn ${{ steps.update1Run.outputs.serverBin }} | .github/helper-bot/disa.exe -s1 .github/helper-bot/rodata.bin > .github/helper-bot/stage1.txt + run: objdump -d --demangle --no-show-raw-insn ${{ steps.update1Run.outputs.serverBin }} | .github/helper-bot/disa.exe -s1 .github/helper-bot/rodata.bin > .github/helper-bot/stage1.txt - name: Process Windows bin (Stage 2) if: steps.update1Run.outputs.needsUpdate run: objdump -d --demangle -M intel ${{ steps.update1Run.outputs.serverWinBin }} | .github/helper-bot/disa.exe -s2 .github/helper-bot/stage1.txt > .github/helper-bot/stage2.txt From e5b40845371359c9d0b47519ed29023a6c1cd146 Mon Sep 17 00:00:00 2001 From: extremeheat Date: Fri, 5 Apr 2024 05:41:21 -0400 Subject: [PATCH 39/50] update --- .github/helper-bot/disa.cpp | 17 +++++++++++++++-- .github/workflows/update-helper.yml | 4 ---- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/.github/helper-bot/disa.cpp b/.github/helper-bot/disa.cpp index f7d4c27c..1862ad4c 100644 --- a/.github/helper-bot/disa.cpp +++ b/.github/helper-bot/disa.cpp @@ -82,6 +82,17 @@ std::string getRoDataStringNT(unsigned int offset) { return std::string(roData + bufferOffset, len); } +std::string fnv64Hex(std::string_view str) { + unsigned long long hash = 0xcbf29ce484222325; + for (size_t i = 0; i < str.size(); i++) { + hash *= 0x100000001b3; + hash ^= str[i]; + } + char buffer[17]; + snprintf(buffer, 17, "%016llx", hash); + return "0x" + std::string(buffer); +} + // // 708c23b: lea 0x1647ede(%rip),%rsi # 86d4120 enum InstructionType { NO_INSTR, MOVABS, MOV, LEA, CALL, OTHER, FUNCTION_START }; @@ -476,10 +487,12 @@ void loadDisassembly(std::string filePath) { auto states = line.substr(statesPos + 15, line.size() - statesPos - 16); if (isRoDataStringNT(lastLoadedAddress)) { auto str = getRoDataStringNT(lastLoadedAddress); - std::cout << "VanillaState\t" << states << "\t" << lastLoadedAddressAbsMovStr << "\t" << str << std::endl; + auto computedHash = fnv64Hex(str); // lastLoadedAddressAbsMovStr can be optimized out + std::cout << "VanillaState\t" << states << "\t" << computedHash << "\t" << str << std::endl; } else if (isRoDataStringNT(lastLastLoadedAddress)) { auto str = getRoDataStringNT(lastLastLoadedAddress); - std::cout << "VanillaState\t" << states << "\t" << lastLoadedAddressAbsMovStr << "\t" << str << std::endl; + auto computedHash = fnv64Hex(str); + std::cout << "VanillaState\t" << states << "\t" << computedHash << "\t" << str << std::endl; } else { // std::cout << "? NOT adding VanillaState\t" << states << " " << lastLoadedAddress << "\t" // << lastLoadedAddressAbsMovStr << std::endl; diff --git a/.github/workflows/update-helper.yml b/.github/workflows/update-helper.yml index 5b5e2854..2c1c1d64 100644 --- a/.github/workflows/update-helper.yml +++ b/.github/workflows/update-helper.yml @@ -67,10 +67,6 @@ jobs: run: | objcopy -O binary -j .rodata ${{ steps.update1Run.outputs.serverBin }} .github/helper-bot/rodata.bin readelf -S ${{ steps.update1Run.outputs.serverBin }} | grep .rodata >> .github/helper-bot/rodata.bin - - run: sha1sum ${{ steps.update1Run.outputs.serverBin }} - - run: sha1sum .github/helper-bot/rodata.bin - - run: llvm-objdump-14 -d --demangle --no-show-raw-insn ${{ steps.update1Run.outputs.serverBin }} > llvm.asm - - run: head -n 20 llvm.asm - name: Process Linux server (Stage 1) if: steps.update1Run.outputs.needsUpdate run: objdump -d --demangle --no-show-raw-insn ${{ steps.update1Run.outputs.serverBin }} | .github/helper-bot/disa.exe -s1 .github/helper-bot/rodata.bin > .github/helper-bot/stage1.txt From 131a55935ad475176da4538e1d29a505fe1cb559 Mon Sep 17 00:00:00 2001 From: extremeheat Date: Fri, 5 Apr 2024 13:23:37 -0400 Subject: [PATCH 40/50] remove stage3, fix rodata address checking in disa --- .github/helper-bot/disa.cpp | 47 +++++++++++++++++++++-------- .github/workflows/update-helper.yml | 3 -- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/.github/helper-bot/disa.cpp b/.github/helper-bot/disa.cpp index 1862ad4c..4141b98e 100644 --- a/.github/helper-bot/disa.cpp +++ b/.github/helper-bot/disa.cpp @@ -60,27 +60,38 @@ void loadRoData(std::string binFile) { fprintf(stderr, "Opened rodata file '%s', size: %d, offset: %d\n", binFile.c_str(), size, roDataOffset); } +// (A-Z, a-z, 0-9, symbols) +bool isValidAsciiChar(char c) { return c >= 'A' && c <= '~'; } bool isAddressInRoData(unsigned int address) { return address >= roDataOffset && address < roDataEnd; } -bool isRoDataStringNT(unsigned int address) { - if (!isAddressInRoData(address)) { +bool isValidRoDataStrAddr(unsigned int address) { + if (!isAddressInRoData(address + 1)) { return false; } auto bufferOffset = address - roDataOffset; - return roData[bufferOffset] != '\0'; + auto c = roData[bufferOffset]; + // Check that c is an ASCII char + return isValidAsciiChar(c); } // Get a null-terminated string from the rodata section std::string getRoDataStringNT(unsigned int offset) { - if (!isRoDataStringNT(offset)) { + if (!isValidRoDataStrAddr(offset)) { return ""; } auto bufferOffset = offset - roDataOffset; int len = 0; - while (roData[bufferOffset + len] != '\0') { + while (isAddressInRoData(offset + len) && roData[bufferOffset + len] != '\0') { len++; } return std::string(roData + bufferOffset, len); } +float getRoDataFloat(unsigned int offset) { + if (!isAddressInRoData(offset)) { + return -0.0f; + } + auto bufferOffset = offset - roDataOffset; + return *(float *)(roData + bufferOffset); +} std::string fnv64Hex(std::string_view str) { unsigned long long hash = 0xcbf29ce484222325; @@ -357,7 +368,7 @@ void loadDisassembly(std::string filePath) { // B1. if we are tracking a block, then print the constant if (!trackingBlock.empty()) { // if line includes '#', then split by the comment and get the comment - if (isRoDataStringNT(lastLoadedAddress)) { + if (isValidRoDataStrAddr(lastLoadedAddress)) { std::string str = getRoDataStringNT(lastLoadedAddress); std::cout << "BlockID\t" << trackingBlock << "\t" << loadedStr << "\t" << str << std::endl; seenBlockIds.push_back(trackingBlock); @@ -414,7 +425,7 @@ void loadDisassembly(std::string filePath) { if (inStateSerializer.size() > 0) { // we are interested in capturing all loaded constants inside the state serializer - if (isRoDataStringNT(instr.commentAddr)) { + if (isValidRoDataStrAddr(instr.commentAddr)) { auto str = getRoDataStringNT(instr.commentAddr); stateEntries[inStateSerializer].push_back(str); } @@ -456,7 +467,7 @@ void loadDisassembly(std::string filePath) { // If we've already seen the block, above check is not needed if (!contains(seenBlockIds, trackingBlock)) { // if line includes '#', then split by the comment and get the comment - if (isRoDataStringNT(lastLoadedAddress)) { + if (isValidRoDataStrAddr(lastLoadedAddress)) { auto str = getRoDataStringNT(lastLoadedAddress); std::cout << "BlockID\t" << trackingBlock << "\t" << "UNK" << "\t" << str << std::endl; } @@ -485,11 +496,11 @@ void loadDisassembly(std::string filePath) { goto finish; } auto states = line.substr(statesPos + 15, line.size() - statesPos - 16); - if (isRoDataStringNT(lastLoadedAddress)) { + if (isValidRoDataStrAddr(lastLoadedAddress)) { auto str = getRoDataStringNT(lastLoadedAddress); auto computedHash = fnv64Hex(str); // lastLoadedAddressAbsMovStr can be optimized out std::cout << "VanillaState\t" << states << "\t" << computedHash << "\t" << str << std::endl; - } else if (isRoDataStringNT(lastLastLoadedAddress)) { + } else if (isValidRoDataStrAddr(lastLastLoadedAddress)) { auto str = getRoDataStringNT(lastLastLoadedAddress); auto computedHash = fnv64Hex(str); std::cout << "VanillaState\t" << states << "\t" << computedHash << "\t" << str << std::endl; @@ -541,10 +552,10 @@ void loadDisassembly(std::string filePath) { ((std::ifstream *)disStream)->close(); delete disStream; } - // Print out the block data for (auto &block : blockData) { - std::cout << "BlockData\t" << block.blockName << "\t" << block.blockClass << "\t" << block.breakTimeAddr << "\t"; + auto flt = getRoDataFloat(block.breakTimeAddr); + std::cout << "BlockData\t" << block.blockName << "\t" << block.blockClass << "\t" << flt << "\t"; for (auto &state : block.stateKeys) { std::cout << state << ","; } @@ -600,6 +611,10 @@ void loadStage1(std::string filePath) { std::string name, id, hash, stateName; split4(line, name, id, hash, stateName); if (name == "VanillaState") { + // Strip the 0x prefix and leading zeros - not insignificant and disassembler may not include them + while (hash.size() > 0 && (hash[0] == '0' || hash[0] == 'x')) { + hash = hash.substr(1, hash.size() - 1); + } stateVariantMap[hash] = {}; } } @@ -671,7 +686,13 @@ void loadDisassembly2(std::string filePath) { } for (auto &entry : stateVariantMap) { - std::cout << "StateVariantData\t" << entry.first << "\t"; + auto hash = entry.first; + // re-add the 0x and leading 0s + while (hash.size() < 16) { + hash = "0" + hash; + } + hash = "0x" + hash; + std::cout << "StateVariantData\t" << hash << "\t"; for (auto &value : entry.second) { std::cout << value << ","; } diff --git a/.github/workflows/update-helper.yml b/.github/workflows/update-helper.yml index 2c1c1d64..9de33a84 100644 --- a/.github/workflows/update-helper.yml +++ b/.github/workflows/update-helper.yml @@ -73,9 +73,6 @@ jobs: - name: Process Windows bin (Stage 2) if: steps.update1Run.outputs.needsUpdate run: objdump -d --demangle -M intel ${{ steps.update1Run.outputs.serverWinBin }} | .github/helper-bot/disa.exe -s2 .github/helper-bot/stage1.txt > .github/helper-bot/stage2.txt - - name: Resolve numbers (Stage 3) - if: steps.update1Run.outputs.needsUpdate - run : node update2.js -s3 working-directory: .github/helper-bot - name: Resolve PDB symbols (Stage 4) if: steps.update1Run.outputs.needsUpdate From 7e437807c03e8409cbfb132da469bb29494b8247 Mon Sep 17 00:00:00 2001 From: extremeheat Date: Fri, 5 Apr 2024 13:29:30 -0400 Subject: [PATCH 41/50] update update3 --- .github/helper-bot/update2.js | 111 ---------------------------------- .github/helper-bot/update3.js | 2 +- 2 files changed, 1 insertion(+), 112 deletions(-) delete mode 100644 .github/helper-bot/update2.js diff --git a/.github/helper-bot/update2.js b/.github/helper-bot/update2.js deleted file mode 100644 index 6947d136..00000000 --- a/.github/helper-bot/update2.js +++ /dev/null @@ -1,111 +0,0 @@ -/* eslint-disable no-tabs */ -const { join } = require('path') -const fs = require('fs') - -// 00ba0080 -function loadRoDataHexDump (filePath) { - const rodataHexDump = fs.readFileSync(filePath) - const buffer = rodataHexDump - const initialBufferOffset = Buffer.from(rodataHexDump.slice(-9, -1).toString('latin1'), 'latin1').readUInt32BE(0) - - function readStringNTFromHexDump (addr) { - const addrNum = typeof addr === 'string' ? parseInt(addr, 16) : addr - const startIx = addrNum - initialBufferOffset - const endIx = buffer.indexOf(0, startIx) - if (endIx === -1) return null - return buffer.toString('latin1', startIx, endIx) - } - function readFloatLEFromHexDump (addr) { - const addrNum = typeof addr === 'string' ? parseInt(addr, 16) : addr - const startIx = addrNum - initialBufferOffset - return buffer.readFloatLE(startIx) - } - function readDoubleLEFromHexDump (addr) { - const addrNum = typeof addr === 'string' ? parseInt(addr, 16) : addr - const startIx = addrNum - initialBufferOffset - return buffer.readDoubleLE(startIx) - } - function readIntLEFromHexDump (addr) { - const addrNum = typeof addr === 'string' ? parseInt(addr, 16) : addr - const startIx = addrNum - initialBufferOffset - return buffer.readInt32LE(startIx) - } - function readI64LEFromHexDump (addr) { - const addrNum = typeof addr === 'string' ? parseInt(addr, 16) : addr - const startIx = addrNum - initialBufferOffset - return buffer.readBigInt64LE(startIx) - } - return { buffer, initialBufferOffset, readStringNTFromHexDump, readFloatLEFromHexDump, readDoubleLEFromHexDump, readIntLEFromHexDump, readI64LEFromHexDump } -} - -function writeStringsTSV (rodataDump, stringsFile) { - const { readStringNTFromHexDump } = loadRoDataHexDump(rodataDump, stringsFile) - - let result = '' - const strings = fs.readFileSync(stringsFile, 'utf8') - for (let i = 0; i < strings.length; i++) { - const endIx = strings.indexOf('\n', i) - // split the sub string by spaces - const subStr = strings.substring(i, endIx) - const split = subStr.split(' ') - const address = split.find(s => s.startsWith('0x')) - const contents = subStr.split('ascii')[1]?.trim() - if (!contents || !contents.match(/^[a-zA-Z0-9_:]+$/) || contents.length > 64) { - if (subStr.includes('utf16le')) { - // unfortunately some ascii strings are incorrectly written as utf16le - const contents16 = subStr.split('utf16le')[1]?.trim() - for (let j = 0; j < contents16.length; j += 1) { - // write each char one by one as 1 length string - const newAddress = `0x${(parseInt(address, 16) + (j * 2)).toString(16).padStart(8, '0')}` - result += `# fromU16LE ${address}\n` - const ntString = readStringNTFromHexDump(newAddress) - if (ntString && ntString.match(/^[a-zA-Z0-9_:]+$/) && ntString.length < 64) { - result += `${newAddress}\t${ntString}\n` - } - } - } - } else { - result += `${address}\t${contents}\n` - } - i = endIx - } - fs.writeFileSync('strings.tsv', result) -} - -if (process.argv.length < 2) { - console.log('Usage: node extract.js ') - process.exit(1) -} - -function postProc (rodataDump, s1file) { - const { readFloatLEFromHexDump } = loadRoDataHexDump(rodataDump) - const stage2 = fs.readFileSync(s1file, 'latin1') - let result = '' - for (const line of stage2.split('\n')) { - const slices = line.split('\t') - if (slices[0] === 'BlockData') { - // console.log(slices) - try { - const float = readFloatLEFromHexDump(parseInt(slices[3])) - console.log('BlockExtraData', slices[1], float) - result += `BlockExtraData\t${slices[1]}\t${float}\n` - } catch (e) { - result += `BlockExtraData\t${slices[1]}\t${0}\n` - } - } - } - fs.writeFileSync('stage3.txt', result) -} - -const stage = process.argv[2] -if (stage === '-s0') { - writeStringsTSV( - join(__dirname, 'rodata.bin'), - join(__dirname, 'strings.txt') - ) -} else if (stage === '-s3') { - postProc( - join(__dirname, 'rodata.bin'), - join(__dirname, 'stage1.txt') - ) -} diff --git a/.github/helper-bot/update3.js b/.github/helper-bot/update3.js index 522f7b58..3f09406f 100644 --- a/.github/helper-bot/update3.js +++ b/.github/helper-bot/update3.js @@ -2,7 +2,7 @@ const fs = require('fs') const github = require('gh-helpers')() async function main () { - const stages = ['stage1.txt', 'stage2.txt', 'stage3.txt', 'stage4.txt'] + const stages = ['stage1.txt', 'stage2.txt', 'stage4.txt'] const allStages = fs.createWriteStream('merged.txt') for (const stage of stages) { allStages.write(fs.readFileSync(stage, 'latin1')) From 4e90c153360f635115a5e8e8b2976c995a6cdd9c Mon Sep 17 00:00:00 2001 From: extremeheat Date: Fri, 5 Apr 2024 14:13:28 -0400 Subject: [PATCH 42/50] update --- .github/helper-bot/disa.cpp | 38 ++++++++++++++++++----------- .github/workflows/update-helper.yml | 1 - 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/.github/helper-bot/disa.cpp b/.github/helper-bot/disa.cpp index 4141b98e..fd730bd9 100644 --- a/.github/helper-bot/disa.cpp +++ b/.github/helper-bot/disa.cpp @@ -69,7 +69,6 @@ bool isValidRoDataStrAddr(unsigned int address) { } auto bufferOffset = address - roDataOffset; auto c = roData[bufferOffset]; - // Check that c is an ASCII char return isValidAsciiChar(c); } @@ -92,7 +91,20 @@ float getRoDataFloat(unsigned int offset) { auto bufferOffset = offset - roDataOffset; return *(float *)(roData + bufferOffset); } - +std::string getRoDataHexDump(unsigned int offset, int len) { + if (!isAddressInRoData(offset)) { + return ""; + } + auto bufferOffset = offset - roDataOffset; + std::string hexDump; + for (int i = 0; i < len; i++) { + char c = roData[bufferOffset + i]; + char buffer[4]; + snprintf(buffer, 4, "%02x", c); + hexDump += buffer; + } + return hexDump; +} std::string fnv64Hex(std::string_view str) { unsigned long long hash = 0xcbf29ce484222325; for (size_t i = 0; i < str.size(); i++) { @@ -163,9 +175,9 @@ void parseAttLine(char *buffer, Instruction &instr) { } } } - instr.asciiOpEnd = buffer + i; // op end here holds the symbol name - // remove the colon + instr.asciiOpEnd = buffer + i; + // remove the trailing colon instr.asciiOpEnd--; } else { bool readingAddress = true; @@ -266,7 +278,6 @@ void parseAttLine(char *buffer, Instruction &instr) { if (instr.asciiCommentStart && instr.asciiCommentEnd) { // Comment Start: [ 86d4120 ] // Comment End: [] - // iterate until the first '<' and then until the first '>' char *asciiCommentAddrStart = instr.asciiCommentStart + 1; char *asciiCommentAddrEnd = nullptr; char *asciiCommentSymbolStart = nullptr; @@ -395,8 +406,7 @@ void loadDisassembly(std::string filePath) { std::cerr << "? Unloaded Block registration: " << line << std::endl; } } else { - currentBlockData = CurrentBlockData{.blockClass = std::string(blockClass)}; - currentBlockData->blockName = trackingBlock; + currentBlockData = CurrentBlockData{.blockName = trackingBlock, .blockClass = std::string(blockClass)}; } } @@ -437,7 +447,8 @@ void loadDisassembly(std::string filePath) { auto sharedNameStr = std::string(sharedName); if (!contains(seenConstants, sharedNameStr)) { seenConstants.push_back(sharedNameStr); - std::cout << "Const\t" << sharedName << "\t" << instr.commentAddr << std::endl; + auto hexDump = getRoDataHexDump(instr.commentAddr, 32); + std::cout << "Const\t" << sharedName << "\t" << instr.commentAddr << "\t" << hexDump << std::endl; } } @@ -473,7 +484,7 @@ void loadDisassembly(std::string filePath) { } } } - // lea/mov are both used to load constants before a call, so we can keep tracking if it's also a mov + // lea/mov are both used to load args before a call, so we can keep tracking if it's also a mov if (instr.type != MOV) { trackingBlock.clear(); } @@ -488,10 +499,10 @@ void loadDisassembly(std::string filePath) { lastLastLoadedAddress = lastLoadedAddress; lastLoadedAddress = instr.commentAddr; } else if (isInGlobalBlock) { - // State Registration? size_t statesPos = line.find("VanillaStates::"); + // State Registration if (statesPos != std::string::npos) { - // ensure there's no + in the symbol + // ensure there's no + offset in the symbol if (line.find("+") != std::string::npos) { goto finish; } @@ -517,7 +528,7 @@ void loadDisassembly(std::string filePath) { if (instr.isFunctionStart) { std::string_view line(buffer); trackingBlock.clear(); - // globals initialization + // globals initialization block if (line.find("_GLOBAL_") != std::string::npos) { isInGlobalBlock = true; } else { @@ -714,7 +725,7 @@ int main(int argc, char **argv) { } std::cout << "Stage: " << stage << std::endl; if (disFile.empty()) { - std::cout << "(waiting for stdin)" << std::endl; + std::cerr << "(waiting for stdin)" << std::endl; } if (stage == "-s1") { loadRoData(file); @@ -723,7 +734,6 @@ int main(int argc, char **argv) { loadStage1(file); loadDisassembly2(disFile); } - printf("Done\n"); std::cerr << "Done" << std::endl; return 0; } diff --git a/.github/workflows/update-helper.yml b/.github/workflows/update-helper.yml index 9de33a84..ccde34d4 100644 --- a/.github/workflows/update-helper.yml +++ b/.github/workflows/update-helper.yml @@ -73,7 +73,6 @@ jobs: - name: Process Windows bin (Stage 2) if: steps.update1Run.outputs.needsUpdate run: objdump -d --demangle -M intel ${{ steps.update1Run.outputs.serverWinBin }} | .github/helper-bot/disa.exe -s2 .github/helper-bot/stage1.txt > .github/helper-bot/stage2.txt - working-directory: .github/helper-bot - name: Resolve PDB symbols (Stage 4) if: steps.update1Run.outputs.needsUpdate run: llvm-pdbutil-14 dump --all ${{ steps.update1Run.outputs.serverWinPdb }} | .github/helper-bot/pdba.exe > .github/helper-bot/stage4.txt From 59901fb43e2cbce821f04f26c6cab1de964276e4 Mon Sep 17 00:00:00 2001 From: extremeheat Date: Tue, 9 Apr 2024 04:52:01 -0400 Subject: [PATCH 43/50] disa: add instruction parser --- .github/helper-bot/disa.cpp | 223 +------------ .github/helper-bot/disa.h | 610 ++++++++++++++++++++++++++++++++++++ 2 files changed, 612 insertions(+), 221 deletions(-) create mode 100644 .github/helper-bot/disa.h diff --git a/.github/helper-bot/disa.cpp b/.github/helper-bot/disa.cpp index fd730bd9..24c5cbb0 100644 --- a/.github/helper-bot/disa.cpp +++ b/.github/helper-bot/disa.cpp @@ -6,37 +6,7 @@ #include #include #include - -void ZeroMemory(char *buffer, int size) { - for (int i = 0; i < size; i++) { - buffer[i] = 0; - } -} - -unsigned int hexStr2Int(const std::string_view &hexStr) { - const char *hexStrC = hexStr.data(); - unsigned int value = 0; - for (size_t i = 0; i < hexStr.size(); i++) { - char c = hexStrC[i]; - if (c >= '0' && c <= '9') { - value = (value << 4) + (c - '0'); - } else if (c >= 'a' && c <= 'f') { - value = (value << 4) + (c - 'a' + 10); - } else if (c >= 'A' && c <= 'F') { - value = (value << 4) + (c - 'A' + 10); - } - } - return value; -} - -unsigned int hexStr2IntLE(const std::string_view &hexStr) { - unsigned int value = hexStr2Int(hexStr); - unsigned int swapped = ((value >> 24) & 0xff) | // move byte 3 to byte 0 - ((value << 8) & 0xff0000) | // move byte 1 to byte 2 - ((value >> 8) & 0xff00) | // move byte 2 to byte 1 - ((value << 24) & 0xff000000); // byte 0 to byte 3 - return swapped; -} +#include "disa.h" char *roData; int roDataOffset; @@ -116,196 +86,6 @@ std::string fnv64Hex(std::string_view str) { return "0x" + std::string(buffer); } -// -// 708c23b: lea 0x1647ede(%rip),%rsi # 86d4120 -enum InstructionType { NO_INSTR, MOVABS, MOV, LEA, CALL, OTHER, FUNCTION_START }; -struct Instruction { - InstructionType type; - bool isFunctionStart; - char *asciiAddressStart; - char *asciiAddressEnd; - char *asciiOpStart; - char *asciiOpEnd; - char *asciiOperandsStart; - char *asciiOperandsEnd; - char *asciiCommentStart; - char *asciiCommentEnd; - unsigned int commentAddr; - char *commentSymbolStart; - char *commentSymbolEnd; -}; -void clearInstruction(Instruction &instr) { - instr.type = NO_INSTR; - instr.isFunctionStart = false; - instr.asciiAddressStart = nullptr; - instr.asciiAddressEnd = nullptr; - instr.asciiOpStart = nullptr; - instr.asciiOpEnd = nullptr; - instr.asciiOperandsStart = nullptr; - instr.asciiOperandsEnd = nullptr; - instr.asciiCommentStart = nullptr; - instr.asciiCommentEnd = nullptr; - instr.commentAddr = 0; - instr.commentSymbolStart = nullptr; - instr.commentSymbolEnd = nullptr; -} -void parseAttLine(char *buffer, Instruction &instr) { - instr.asciiAddressStart = buffer; - if (buffer[0] == ' ') { - instr.isFunctionStart = false; - } else { - instr.isFunctionStart = true; - } - - if (instr.isFunctionStart) { - // 0000000002c44530 : - bool readingAddress = true; - bool readingSymbol = false; - int i = 0; - for (;; i++) { - auto c = buffer[i]; - if (c == '\0') - break; - if (c == ' ' || c == '\t') { - if (readingAddress) { - readingAddress = false; - readingSymbol = true; - instr.asciiAddressEnd = buffer + i; - instr.asciiOpStart = buffer + i + 1; - } - } - } - // op end here holds the symbol name - instr.asciiOpEnd = buffer + i; - // remove the trailing colon - instr.asciiOpEnd--; - } else { - bool readingAddress = true; - bool readingOp = false; - bool readingOperands = false; - bool readingComment = false; - for (int i = 0; true; i++) { - auto c = buffer[i]; - if (c == '\0') - break; - - if (readingAddress) { - for (int j = i; true; j++) { - auto c = buffer[j]; - if (c == '\0') - break; - if (c == ':') { - readingAddress = false; - readingOp = true; - instr.asciiAddressEnd = buffer + j; - i = j; - break; - } - } - } else if (readingOp) { - for (int j = i; true; j++) { - auto c = buffer[j]; - if (c == '\0') - break; - if (c == ' ' || c == '\t') { - if (instr.asciiOpStart) { - readingOp = false; - readingOperands = true; - instr.asciiOpEnd = buffer + j; - i = j; - break; - } - } else if (!instr.asciiOpStart) { - instr.asciiOpStart = buffer + j; - } - } - } else if (readingOperands) { - for (int j = i; true; j++) { - auto c = buffer[j]; - if (c == '#' || c == '\0') { - readingOperands = false; - readingComment = true; - instr.asciiOperandsEnd = buffer + j; - i = j; - break; - } else if (!(c == ' ' || c == '\t')) { - if (!instr.asciiOperandsStart) { - instr.asciiOperandsStart = buffer + j; - } - } - } - } else if (readingComment) { - for (int j = i; true; j++) { - auto c = buffer[j]; - if (c == '\0') { - readingComment = false; - instr.asciiCommentEnd = buffer + j; - i = j; - break; - } else if (!instr.asciiCommentStart) { - instr.asciiCommentStart = buffer + j; - } - } - } - } - } - - // Sanity check: make sure we have at least an op start and end (this also covers functions) - if (!instr.asciiOpStart || !instr.asciiOpEnd) { - instr.type = NO_INSTR; - instr.isFunctionStart = false; - return; - } - - if (instr.isFunctionStart) { - instr.type = FUNCTION_START; - return; - } - - auto op = instr.asciiOpStart; - if (op[0] == 'm' && op[1] == 'o' && op[2] == 'v' && op[3] == 'a' && op[4] == 'b' && op[5] == 's') { - instr.type = MOVABS; - } else if (op[0] == 'm' && op[1] == 'o' && op[2] == 'v') { - instr.type = MOV; - } else if (op[0] == 'l' && op[1] == 'e' && op[2] == 'a') { - instr.type = LEA; - } else if (op[0] == 'c' && op[1] == 'a' && op[2] == 'l' && op[3] == 'l') { - instr.type = CALL; - } else { - instr.type = OTHER; - } - - if (instr.asciiCommentStart && instr.asciiCommentEnd) { - // Comment Start: [ 86d4120 ] - // Comment End: [] - char *asciiCommentAddrStart = instr.asciiCommentStart + 1; - char *asciiCommentAddrEnd = nullptr; - char *asciiCommentSymbolStart = nullptr; - char *asciiCommentSymbolEnd = nullptr; - for (int i = 0; true; i++) { - auto c = asciiCommentAddrStart[i]; - if (c == '\0') - break; - if (c == '<' && !asciiCommentAddrEnd) { - asciiCommentAddrEnd = asciiCommentAddrStart + i; - asciiCommentSymbolStart = asciiCommentAddrStart + i + 1; - } else if (c == '>') { - asciiCommentSymbolEnd = asciiCommentAddrStart + i; - break; - } - } - if (asciiCommentAddrEnd && asciiCommentSymbolStart && asciiCommentSymbolEnd) { - instr.commentAddr = - hexStr2Int(std::string_view(asciiCommentAddrStart, asciiCommentAddrEnd - asciiCommentAddrStart)); - instr.commentSymbolStart = asciiCommentSymbolStart; - instr.commentSymbolEnd = asciiCommentSymbolEnd; - } - } -} -// - -// void parseIntelLine() {} - struct CurrentBlockData { std::string blockName; std::string blockClass; @@ -366,6 +146,7 @@ void loadDisassembly(std::string filePath) { if (instr.type == NO_INSTR) { goto finish; } + registerProcessInstruction(instr); // movabs $0x116b2c0, %rbx -> move the address to rbx if (instr.type == MOVABS) { diff --git a/.github/helper-bot/disa.h b/.github/helper-bot/disa.h new file mode 100644 index 00000000..f97295c5 --- /dev/null +++ b/.github/helper-bot/disa.h @@ -0,0 +1,610 @@ +#include +#include +#include +#include + +typedef unsigned long long int u64; + +void ZeroMemory(char *buffer, int size) { + for (int i = 0; i < size; i++) { + buffer[i] = 0; + } +} + +void StringCopyInto(char *dest, const char *src) { + while (*src) { + *dest = *src; + dest++; + src++; + } + *dest = 0; +} +void StringCopyInto(char *dest, const char *src, int size, int max) { + size = size < max ? size : max; + ZeroMemory(dest, size); + for (int i = 0; i < size; i++) { + dest[i] = src[i]; + } + dest[size] = 0; +} +void StringStartsWith(std::string_view &str, std::string_view &prefix) { + if (str.size() < prefix.size()) { + return; + } + for (size_t i = 0; i < prefix.size(); i++) { + if (str[i] != prefix[i]) { + return; + } + } +} + +unsigned int hexStr2Int(const std::string_view &hexStr) { + const char *hexStrC = hexStr.data(); + unsigned int value = 0; + for (size_t i = 0; i < hexStr.size(); i++) { + char c = hexStrC[i]; + if (c >= '0' && c <= '9') { + value = (value << 4) + (c - '0'); + } else if (c >= 'a' && c <= 'f') { + value = (value << 4) + (c - 'a' + 10); + } else if (c >= 'A' && c <= 'F') { + value = (value << 4) + (c - 'A' + 10); + } + } + return value; +} + +unsigned int hexStr2IntLE(const std::string_view &hexStr) { + unsigned int value = hexStr2Int(hexStr); + unsigned int swapped = ((value >> 24) & 0xff) | // move byte 3 to byte 0 + ((value << 8) & 0xff0000) | // move byte 1 to byte 2 + ((value >> 8) & 0xff00) | // move byte 2 to byte 1 + ((value << 24) & 0xff000000); // byte 0 to byte 3 + return swapped; +} + +u64 hexStr2Int64(const std::string_view &hexStr) { + const char *hexStrC = hexStr.data(); + u64 value = 0; + for (size_t i = 0; i < hexStr.size(); i++) { + char c = hexStrC[i]; + if (c >= '0' && c <= '9') { + value = (value << 4) + (c - '0'); + } else if (c >= 'a' && c <= 'f') { + value = (value << 4) + (c - 'a' + 10); + } else if (c >= 'A' && c <= 'F') { + value = (value << 4) + (c - 'A' + 10); + } + } + return value; +} + +// INSTR PARSE +// 708c23b: lea 0x1647ede(%rip),%rsi # 86d4120 +enum InstructionType { NO_INSTR, MOVABS, MOV, LEA, CALL, OTHER, FUNCTION_START }; +struct Instruction { + InstructionType type; + bool isFunctionStart; + char *asciiAddressStart; + char *asciiAddressEnd; + char *asciiOpStart; + char *asciiOpEnd; + char *asciiOperandsStart; + char *asciiOperandsEnd; + char *asciiCommentStart; + char *asciiCommentEnd; + unsigned int commentAddr; + char *commentSymbolStart; + char *commentSymbolEnd; +}; +void clearInstruction(Instruction &instr) { + instr.type = NO_INSTR; + instr.isFunctionStart = false; + instr.asciiAddressStart = nullptr; + instr.asciiAddressEnd = nullptr; + instr.asciiOpStart = nullptr; + instr.asciiOpEnd = nullptr; + instr.asciiOperandsStart = nullptr; + instr.asciiOperandsEnd = nullptr; + instr.asciiCommentStart = nullptr; + instr.asciiCommentEnd = nullptr; + instr.commentAddr = 0; + instr.commentSymbolStart = nullptr; + instr.commentSymbolEnd = nullptr; +} +void parseAttLine(char *buffer, Instruction &instr) { + instr.asciiAddressStart = buffer; + if (buffer[0] == ' ') { + instr.isFunctionStart = false; + } else { + instr.isFunctionStart = true; + } + + if (instr.isFunctionStart) { + // 0000000002c44530 : + bool readingAddress = true; + bool readingSymbol = false; + int i = 0; + for (;; i++) { + auto c = buffer[i]; + if (c == '\0') + break; + if (c == ' ' || c == '\t') { + if (readingAddress) { + readingAddress = false; + readingSymbol = true; + instr.asciiAddressEnd = buffer + i; + instr.asciiOpStart = buffer + i + 1; + } + } + } + // op end here holds the symbol name + instr.asciiOpEnd = buffer + i; + // remove the trailing colon + instr.asciiOpEnd--; + } else { + bool readingAddress = true; + bool readingOp = false; + bool readingOperands = false; + bool readingComment = false; + for (int i = 0; true; i++) { + auto c = buffer[i]; + if (c == '\0') + break; + + if (readingAddress) { + for (int j = i; true; j++) { + auto c = buffer[j]; + if (c == '\0') + break; + if (c == ':') { + readingAddress = false; + readingOp = true; + instr.asciiAddressEnd = buffer + j; + i = j; + break; + } + } + } else if (readingOp) { + for (int j = i; true; j++) { + auto c = buffer[j]; + if (c == '\0') + break; + if (c == ' ' || c == '\t') { + if (instr.asciiOpStart) { + readingOp = false; + readingOperands = true; + instr.asciiOpEnd = buffer + j; + i = j; + break; + } + } else if (!instr.asciiOpStart) { + instr.asciiOpStart = buffer + j; + } + } + } else if (readingOperands) { + for (int j = i; true; j++) { + auto c = buffer[j]; + if (c == '#' || c == '\0') { + readingOperands = false; + readingComment = true; + instr.asciiOperandsEnd = buffer + j; + i = j; + break; + } else if (!(c == ' ' || c == '\t')) { + if (!instr.asciiOperandsStart) { + instr.asciiOperandsStart = buffer + j; + } + } + } + } else if (readingComment) { + for (int j = i; true; j++) { + auto c = buffer[j]; + if (c == '\0') { + readingComment = false; + instr.asciiCommentEnd = buffer + j; + i = j; + break; + } else if (!instr.asciiCommentStart) { + instr.asciiCommentStart = buffer + j; + } + } + } + } + } + + // Sanity check: make sure we have at least an op start and end (this also covers functions) + if (!instr.asciiOpStart || !instr.asciiOpEnd) { + instr.type = NO_INSTR; + instr.isFunctionStart = false; + return; + } + + if (instr.isFunctionStart) { + instr.type = FUNCTION_START; + return; + } + + auto op = instr.asciiOpStart; + if (op[0] == 'm' && op[1] == 'o' && op[2] == 'v' && op[3] == 'a' && op[4] == 'b' && op[5] == 's') { + instr.type = MOVABS; + } else if (op[0] == 'm' && op[1] == 'o' && op[2] == 'v') { + instr.type = MOV; + } else if (op[0] == 'l' && op[1] == 'e' && op[2] == 'a') { + instr.type = LEA; + } else if (op[0] == 'c' && op[1] == 'a' && op[2] == 'l' && op[3] == 'l') { + instr.type = CALL; + } else { + instr.type = OTHER; + } + + if (instr.asciiCommentStart && instr.asciiCommentEnd) { + // Comment Start: [ 86d4120 ] + // Comment End: [] + char *asciiCommentAddrStart = instr.asciiCommentStart + 1; + char *asciiCommentAddrEnd = nullptr; + char *asciiCommentSymbolStart = nullptr; + char *asciiCommentSymbolEnd = nullptr; + for (int i = 0; true; i++) { + auto c = asciiCommentAddrStart[i]; + if (c == '\0') + break; + if (c == '<' && !asciiCommentAddrEnd) { + asciiCommentAddrEnd = asciiCommentAddrStart + i; + asciiCommentSymbolStart = asciiCommentAddrStart + i + 1; + } else if (c == '>') { + asciiCommentSymbolEnd = asciiCommentAddrStart + i; + break; + } + } + if (asciiCommentAddrEnd && asciiCommentSymbolStart && asciiCommentSymbolEnd) { + instr.commentAddr = + hexStr2Int(std::string_view(asciiCommentAddrStart, asciiCommentAddrEnd - asciiCommentAddrStart)); + instr.commentSymbolStart = asciiCommentSymbolStart; + instr.commentSymbolEnd = asciiCommentSymbolEnd; + } + } +} +void instructionReadOperands(Instruction &instr, std::string_view &a, std::string_view &b, std::string_view &c) { + if (!instr.asciiOperandsStart || !instr.asciiOperandsEnd) { + return; + } + std::string_view operand = + std::string_view(instr.asciiOperandsStart, instr.asciiOperandsEnd - instr.asciiOperandsStart); + + // split by first comma + size_t commaPos1 = operand.find(','); + if (commaPos1 != std::string::npos) { + a = operand.substr(0, commaPos1); + std::string_view remaining = operand.substr(commaPos1 + 1); + + // split by second comma + size_t commaPos2 = remaining.find(','); + if (commaPos2 != std::string::npos) { + b = remaining.substr(0, commaPos2); + c = remaining.substr(commaPos2 + 1); + } else { + b = remaining; + } + } else { + a = operand; + } +} +// END INSTR PARSE + +/* +rax - register a extended +rbx - register b extended +rcx - register c extended +rdx - register d extended +rbp - register base pointer (start of stack) +rsp - register stack pointer (current location in stack, growing downwards) +rsi - register source index (source for data copies) +rdi - register destination index (destination for data copies) +*/ + +const int MAX_SYMBOL_SIZE = 63; +union RegisterVal { + enum RegisterDataType { RDTGeneric, RDTVanillaState, RDTBlockTypeID }; + u64 value; + char symbolValue[MAX_SYMBOL_SIZE + 1]; +}; +struct RegisterState { + RegisterVal rax; // + RegisterVal rbx; // + RegisterVal rcx; // arg4 + RegisterVal rdx; // arg3 + RegisterVal rbp; // start of stack + RegisterVal rsp; // current location in stack, growing downwards + RegisterVal rsi; // register source index (source for data copies) ; arg2 + RegisterVal rdi; // register destination index (destination for data copies) ; arg1 + RegisterVal r8; + RegisterVal r9; + RegisterVal r10; + RegisterVal r11; + RegisterVal r12; + RegisterVal r13; + RegisterVal r14; + RegisterVal r15; + RegisterVal rip; + RegisterVal rflags; + RegisterVal xmm0; + RegisterVal xmm1; + RegisterVal xmm2; + RegisterVal xmm3; +}; +RegisterState g_registerState; +enum Register { + REG_UNKNOWN, + RAX, + RBX, + RCX, + RDX, + RBP, + RSP, + RSI, + RDI, + R8, + R9, + R10, + R11, + R12, + R13, + R14, + R15, + RIP, + RFLAGS, + XMM0, + XMM1, + XMM2, + XMM3 +}; + +void registerClearState() { g_registerState = RegisterState{}; } + +#define STR_STARTS_WITH2(str, other) (str[0] == other[0] && str[1] == other[1]) +#define STR_STARTS_WITH3(str, other) (str[0] == other[0] && str[1] == other[1] && str[2] == other[2]) +#define STR_STARTS_WITH4(s, o) (s[0] == o[0] && s[1] == o[1] && s[2] == o[2] && s[3] == o[3]) +#define STR_STARTS_WITH5(s, o) (s[0] == o[0] && s[1] == o[1] && s[2] == o[2] && s[3] == o[3] && s[4] == o[4]) + +Register registerGetType(std::string_view str) { + // in AT&T syntax, registers are prefixed with % + std::string_view reg = str[0] == '%' ? str.substr(1) : str; + // std::cout << "Reading register: [" << reg << "]" << std::endl; + // clang-format off + if (STR_STARTS_WITH3(reg, "rax")) return RAX; + if (STR_STARTS_WITH3(reg, "rbx")) return RBX; + if (STR_STARTS_WITH3(reg, "rcx")) return RCX; + if (STR_STARTS_WITH3(reg, "rdx")) return RDX; + if (STR_STARTS_WITH3(reg, "rbp")) return RBP; + if (STR_STARTS_WITH3(reg, "rsp")) return RSP; + if (STR_STARTS_WITH3(reg, "rsi")) return RSI; + if (STR_STARTS_WITH3(reg, "rdi")) return RDI; + if (STR_STARTS_WITH2(reg, "r8")) return R8; + if (STR_STARTS_WITH2(reg, "r9")) return R9; + if (STR_STARTS_WITH3(reg, "r10")) return R10; + if (STR_STARTS_WITH3(reg, "r11")) return R11; + if (STR_STARTS_WITH3(reg, "r12")) return R12; + if (STR_STARTS_WITH3(reg, "r13")) return R13; + if (STR_STARTS_WITH3(reg, "r14")) return R14; + if (STR_STARTS_WITH3(reg, "r15")) return R15; + if (STR_STARTS_WITH3(reg, "rip")) return RIP; + if (STR_STARTS_WITH5(reg, "rflag")) return RFLAGS; + if (STR_STARTS_WITH4(reg, "xmm0")) return XMM0; + if (STR_STARTS_WITH4(reg, "xmm1")) return XMM1; + if (STR_STARTS_WITH4(reg, "xmm2")) return XMM2; + if (STR_STARTS_WITH4(reg, "xmm3")) return XMM3; + // clang-format on + return REG_UNKNOWN; +} + +// The first four integer or pointer parameters are passed in the first four general-purpose registers, rdi, rsi, rdx, +// and rcx. The first four floating-point parameters are passed in the first four SSE registers, xmm0-xmm3. +void registerSwap(RegisterVal &a, RegisterVal &b) { + RegisterVal temp = a; + a = b; + b = temp; +} + +RegisterVal registerGetArgFloat(int index) { + switch (index) { + case 0: + return g_registerState.xmm0; + case 1: + return g_registerState.xmm1; + case 2: + return g_registerState.xmm2; + case 3: + return g_registerState.xmm3; + default: + return RegisterVal{}; + } +} + +RegisterVal registerGetArgInt(int index) { + switch (index) { + case 0: + return g_registerState.rdi; + case 1: + return g_registerState.rsi; + case 2: + return g_registerState.rdx; + case 3: + return g_registerState.rcx; + default: + return RegisterVal{}; + } +} + +#define REG_CASE(U, L) \ + case U: { \ + /*std::cout << "Setting " << #L << " to " << value << std::endl;*/ \ + g_registerState.L.value = value; \ + if (hasCommentSym) \ + StringCopyInto(g_registerState.L.symbolValue, comment.data(), comment.size(), MAX_SYMBOL_SIZE); \ + return &g_registerState.L; \ + } + +RegisterVal *registerSetVal(Register ®, u64 value, bool hasCommentSym, std::string_view comment) { + // std::cout << "Setting Register: " << reg << " to " << value << std::endl; + switch (reg) { + REG_CASE(RAX, rax) + REG_CASE(RBX, rbx) + REG_CASE(RCX, rcx) + REG_CASE(RDX, rdx) + REG_CASE(RBP, rbp) + REG_CASE(RSP, rsp) + REG_CASE(RSI, rsi) + REG_CASE(RDI, rdi) + REG_CASE(R8, r8) + REG_CASE(R9, r9) + REG_CASE(R10, r10) + REG_CASE(R11, r11) + REG_CASE(R12, r12) + REG_CASE(R13, r13) + REG_CASE(R14, r14) + REG_CASE(R15, r15) + REG_CASE(RIP, rip) + REG_CASE(RFLAGS, rflags) + REG_CASE(XMM0, xmm0) + REG_CASE(XMM1, xmm1) + REG_CASE(XMM2, xmm2) + REG_CASE(XMM3, xmm3) + default: + break; + } + return nullptr; +} + +#undef REG_CASE + +void registerCopy(RegisterVal &a, RegisterVal &intoB) { + // copy the value + intoB.value = a.value; + // copy the symbol + StringCopyInto(intoB.symbolValue, a.symbolValue, MAX_SYMBOL_SIZE, MAX_SYMBOL_SIZE); +} + +// clang-format off +#define REG_MOVE_CASE(UFROM, FROM) \ + case UFROM: \ + switch (intoRegSlot) { \ + case RAX: registerCopy(g_registerState.FROM, g_registerState.rax); return &g_registerState.rax; \ + case RBX: registerCopy(g_registerState.FROM, g_registerState.rbx); return &g_registerState.rbx; \ + case RCX: registerCopy(g_registerState.FROM, g_registerState.rcx); return &g_registerState.rcx; \ + case RDX: registerCopy(g_registerState.FROM, g_registerState.rdx); return &g_registerState.rdx; \ + case RBP: registerCopy(g_registerState.FROM, g_registerState.rbp); return &g_registerState.rbp; \ + case RSP: registerCopy(g_registerState.FROM, g_registerState.rsp); return &g_registerState.rsp; \ + case RSI: registerCopy(g_registerState.FROM, g_registerState.rsi); return &g_registerState.rsi; \ + case RDI: registerCopy(g_registerState.FROM, g_registerState.rdi); return &g_registerState.rdi; \ + case R8: registerCopy(g_registerState.FROM, g_registerState.r8); return &g_registerState.r8; \ + case R9: registerCopy(g_registerState.FROM, g_registerState.r9); return &g_registerState.r9; \ + case R10: registerCopy(g_registerState.FROM, g_registerState.r10); return &g_registerState.r10; \ + case R11: registerCopy(g_registerState.FROM, g_registerState.r11); return &g_registerState.r11; \ + case R12: registerCopy(g_registerState.FROM, g_registerState.r12); return &g_registerState.r12; \ + case R13: registerCopy(g_registerState.FROM, g_registerState.r13); return &g_registerState.r13; \ + case R14: registerCopy(g_registerState.FROM, g_registerState.r14); return &g_registerState.r14; \ + case R15: registerCopy(g_registerState.FROM, g_registerState.r15); return &g_registerState.r15; \ + case RIP: registerCopy(g_registerState.FROM, g_registerState.rip); return &g_registerState.rip; \ + case RFLAGS: registerCopy(g_registerState.FROM, g_registerState.rflags); return &g_registerState.rflags; \ + case XMM0: registerCopy(g_registerState.FROM, g_registerState.xmm0); return &g_registerState.xmm0; \ + case XMM1: registerCopy(g_registerState.FROM, g_registerState.xmm1); return &g_registerState.xmm1; \ + case XMM2: registerCopy(g_registerState.FROM, g_registerState.xmm2); return &g_registerState.xmm2; \ + case XMM3: registerCopy(g_registerState.FROM, g_registerState.xmm3); return &g_registerState.xmm3; \ + default: break; \ + } \ + break; +// clang-format on + +// Register holds an integer. It does not hold the value of the register itself. No recursion! +RegisterVal *registerMove(Register &fromRegSlot, Register &intoRegSlot) { + // printf("Moving Register: %d into Register: %d\n", fromRegSlot, intoRegSlot); + switch (fromRegSlot) { + REG_MOVE_CASE(RAX, rax) + REG_MOVE_CASE(RBX, rbx) + REG_MOVE_CASE(RCX, rcx) + REG_MOVE_CASE(RDX, rdx) + REG_MOVE_CASE(RBP, rbp) + REG_MOVE_CASE(RSP, rsp) + REG_MOVE_CASE(RSI, rsi) + REG_MOVE_CASE(RDI, rdi) + REG_MOVE_CASE(R8, r8) + REG_MOVE_CASE(R9, r9) + REG_MOVE_CASE(R10, r10) + REG_MOVE_CASE(R11, r11) + REG_MOVE_CASE(R12, r12) + REG_MOVE_CASE(R13, r13) + REG_MOVE_CASE(R14, r14) + REG_MOVE_CASE(R15, r15) + REG_MOVE_CASE(RIP, rip) + REG_MOVE_CASE(RFLAGS, rflags) + REG_MOVE_CASE(XMM0, xmm0) + REG_MOVE_CASE(XMM1, xmm1) + REG_MOVE_CASE(XMM2, xmm2) + REG_MOVE_CASE(XMM3, xmm3) + default: + break; + } + return nullptr; +} + +// We are really only interested in MOV / MOVABS, and LEA instructions. +void registerProcessInstruction(Instruction &instr) { + if (instr.type == FUNCTION_START) { + registerClearState(); + return; + } else if (instr.type == NO_INSTR) { + return; + } + + std::string_view operand1, operand2, operand3; + instructionReadOperands(instr, operand1, operand2, operand3); + + // std::cout << "Operand1: " << operand1 << std::endl; + // std::cout << "Operand2: " << operand2 << std::endl; + // std::cout << "Operand3: " << operand3 << std::endl; + + std::string_view commentSymbol; + bool hasCommentSymbol = instr.commentSymbolStart && instr.commentSymbolEnd; + if (hasCommentSymbol) { + commentSymbol = std::string_view(instr.commentSymbolStart, instr.commentSymbolEnd - instr.commentSymbolStart); + // std::cout << "Reading Comment Symbol: " << commentSymbol << std::endl; + } + + switch (instr.type) { + case MOVABS: { + // movabs $0x116b2c0, %rbx + // a1 is the value, a2 is the register + Register a1 = registerGetType(operand2); + u64 a2 = hexStr2Int64(operand1); + registerSetVal(a1, a2, hasCommentSymbol, commentSymbol); + break; + } + case MOV: { + // mov %rdi,%rax + // a1 is the source register, a2 is the destination register + Register a1 = registerGetType(operand1); + Register a2 = registerGetType(operand2); + RegisterVal *reg = registerMove(a1, a2); + if (hasCommentSymbol && reg) { + StringCopyInto(reg->symbolValue, commentSymbol.data(), commentSymbol.size(), MAX_SYMBOL_SIZE); + } + break; + } + case LEA: { + // lea 0x1647ede(%rip),%rsi # 86d4120 + // a1 is the address, a2 is the register. The address typically is resolved to a symbol by objdump disassembler, so + // we can use it instead of a relative address. + Register intoReg = registerGetType(operand2); + RegisterVal *rv = registerSetVal(intoReg, instr.commentAddr, hasCommentSymbol, commentSymbol); + break; + } + case CALL: + case OTHER: + break; + case FUNCTION_START: + break; + default: + break; + } +} \ No newline at end of file From 27ef025f664048fc5fee81692fbaecaf15f51490 Mon Sep 17 00:00:00 2001 From: extremeheat Date: Thu, 11 Apr 2024 08:56:21 -0400 Subject: [PATCH 44/50] helper-bot: update disa/pdba for symbol/trait extraction Not yet enabled in workflow, but could be in future --- .github/helper-bot/.clang-format | 4 +- .github/helper-bot/disa.cpp | 151 ++++++++++++++++++++++++++----- .github/helper-bot/disa.h | 36 +++++++- .github/helper-bot/pdba.cpp | 96 ++++++++++++++++++-- 4 files changed, 253 insertions(+), 34 deletions(-) diff --git a/.github/helper-bot/.clang-format b/.github/helper-bot/.clang-format index ab45593a..97f01a4e 100644 --- a/.github/helper-bot/.clang-format +++ b/.github/helper-bot/.clang-format @@ -1,2 +1,4 @@ ColumnLimit: 120 -SortIncludes: false \ No newline at end of file +SortIncludes: false +BreakStringLiterals: false +AllowShortFunctionsOnASingleLine: Empty \ No newline at end of file diff --git a/.github/helper-bot/disa.cpp b/.github/helper-bot/disa.cpp index 24c5cbb0..c43350d7 100644 --- a/.github/helper-bot/disa.cpp +++ b/.github/helper-bot/disa.cpp @@ -31,8 +31,12 @@ void loadRoData(std::string binFile) { } // (A-Z, a-z, 0-9, symbols) -bool isValidAsciiChar(char c) { return c >= 'A' && c <= '~'; } -bool isAddressInRoData(unsigned int address) { return address >= roDataOffset && address < roDataEnd; } +bool isValidAsciiChar(char c) { + return c >= 'A' && c <= '~'; +} +bool isAddressInRoData(unsigned int address) { + return address >= roDataOffset && address < roDataEnd; +} bool isValidRoDataStrAddr(unsigned int address) { if (!isAddressInRoData(address + 1)) { return false; @@ -191,6 +195,25 @@ void loadDisassembly(std::string filePath) { } } + size_t addStatePos = line.find("::addState(BlockState"); + if (addStatePos != std::string::npos) { + if (currentBlockData.has_value()) { + auto arg2 = registerGetArgInt(1); + if (arg2.symbolValue[0] && !contains(currentBlockData->stateKeys, arg2.symbolValue)) { + // VanillaStates::UpdateBit + auto symbol = std::string(arg2.symbolValue); + size_t statePos = symbol.find("::"); + if (statePos != std::string::npos) { + auto stateName = symbol.substr(statePos + 2); + if (stateName.find("::") != std::string::npos) { + stateName = stateName.substr(stateName.find("::") + 2); + } + currentBlockData->stateKeys.push_back(stateName); + } + } + } + } + if (currentBlockData.has_value()) { // 74061b8: callq 6bd9f60 size_t destroyPos = line.find("setDestroyTime"); @@ -238,20 +261,6 @@ void loadDisassembly(std::string filePath) { trackingBlock = line.substr(pos + 21, line.size() - pos - 22); } - if (currentBlockData.has_value()) { - // 758eaa3: lea 0x1145dae(%rip),%rsi # 86d4858 - size_t statesPos = line.find("VanillaStates::"); - if (statesPos != std::string::npos) { - // ensure there's no + in the symbol - if (line.find("+") != std::string::npos) { - continue; - } - auto end = line.find(">"); - auto states = line.substr(statesPos + 15, line.size() - statesPos - 16); - auto statesStr = std::string(states); - currentBlockData->stateKeys.push_back(statesStr); - } - } } } else { // B1. cont. Sometimes the movabs with hash is not after 2x lea ops, so we dump what we have and continue @@ -281,13 +290,15 @@ void loadDisassembly(std::string filePath) { lastLoadedAddress = instr.commentAddr; } else if (isInGlobalBlock) { size_t statesPos = line.find("VanillaStates::"); + size_t altStatePos = line.find("BuiltInBlockStates::"); // State Registration - if (statesPos != std::string::npos) { + if ((statesPos != std::string::npos) || (altStatePos != std::string::npos)) { // ensure there's no + offset in the symbol if (line.find("+") != std::string::npos) { goto finish; } - auto states = line.substr(statesPos + 15, line.size() - statesPos - 16); + auto states = statesPos != std::string::npos ? line.substr(statesPos + 15, line.size() - statesPos - 16) + : line.substr(altStatePos, line.size() - altStatePos - 1); if (isValidRoDataStrAddr(lastLoadedAddress)) { auto str = getRoDataStringNT(lastLoadedAddress); auto computedHash = fnv64Hex(str); // lastLoadedAddressAbsMovStr can be optimized out @@ -364,6 +375,10 @@ void loadDisassembly(std::string filePath) { } } +// STAGE 2 + +#define STR_INCLUDES(haystack, needle) (haystack.find(needle) != std::string::npos) + // StateHash -> integer data for this state (like number of variants) std::map> stateVariantMap; @@ -378,6 +393,8 @@ void split4(std::string_view line, std::string &a, std::string &b, std::string & if (pos3 != std::string::npos) { c = std::string(line.substr(pos2 + 1, pos3 - pos2 - 1)); d = std::string(line.substr(pos3 + 1, line.size() - pos3 - 1)); + } else { + c = std::string(line.substr(pos2 + 1, line.size() - pos2 - 1)); } } } @@ -411,6 +428,48 @@ void loadStage1(std::string filePath) { } } } + fprintf(stderr, "Loaded %d state variants from stage1\n", stateVariantMap.size()); +} + +std::map symbolMap; + +void loadStage4(std::string filePath) { + // load stage1 which is the output of above loadDisassembly function + std::ifstream stage4Stream(filePath, std::ios::binary); + if (!stage4Stream.is_open()) { + std::cerr << "Failed to open file: " << filePath << std::endl; + return; + } + // split by tabs + const int bufferSize = 1024 * 64; + char buffer[bufferSize]; + while (stage4Stream.getline(buffer, bufferSize)) { + std::string_view line(buffer); + size_t pos = line.find("\t"); + if (pos != std::string::npos) { + // WSYM Address SymbolNams + // WSYM 0x6bcbbe2ee1f42f72 BlockTrait::Something + std::string name, address, symbolName, _; + split4(line, name, address, symbolName, _); + if (name == "WSYM") { + symbolMap[address] = symbolName; + } + } + } + fprintf(stderr, "Loaded %d symbols from stage4\n", symbolMap.size()); +} + +bool haveSymbolForAddress(std::string address) { + return symbolMap.find(address) != symbolMap.end(); +} +bool haveSymbolForAddress(std::string_view address) { + return haveSymbolForAddress(std::string(address)); +} +std::string getSymbolForAddress(std::string address) { + return symbolMap[address]; +} +std::string getSymbolForAddress(std::string_view address) { + return getSymbolForAddress(std::string(address)); } void loadDisassembly2(std::string filePath) { @@ -429,18 +488,22 @@ void loadDisassembly2(std::string filePath) { const int bufferSize = 1024 * 64; char buffer[bufferSize]{0}; + std::string trackingBlock; + // if trackingBlockFoundReg is > 0, we're tracking a block. We continue tracking for n instructions. + int trackingBlockFoundReg = 0; + bool inMovInstruction = false; // 140064b38: 48 c7 05 ad 7f 95 02 mov QWORD PTR [rip+0x2957fad],0x4 # 0x1429bcaf0 while (disStream->getline(buffer, bufferSize)) { - if (buffer[36] == 'm' && buffer[37] == 'o' && buffer[38] == 'v' && buffer[39] == ' ') { + if (STR_STARTS_WITH4(&buffer[36], "mov ")) { inMovInstruction = true; } else if (buffer[36] != ' ' && buffer[36] != '\0') { // if the instruction is not a continuation of the previous instruction inMovInstruction = false; } + // now we are looking for the state variants... first look for movabs - if (buffer[36] == 'm' && buffer[37] == 'o' && buffer[38] == 'v' && buffer[39] == 'a' && buffer[40] == 'b' && - buffer[41] == 's') { + if (STR_STARTS_WITH4(&buffer[36], "movabs")) { std::string_view line(buffer); for (auto &entry : stateVariantMap) { @@ -452,6 +515,50 @@ void loadDisassembly2(std::string filePath) { } } + if (STR_STARTS_WITH4(&buffer[36], "lea ")) { + std::string_view line(buffer); + size_t addressPos = line.find(" # "); + if (addressPos != std::string::npos) { + auto addressStr = line.substr(addressPos + 5); + if (haveSymbolForAddress(addressStr)) { + auto symbol = getSymbolForAddress(addressStr); + size_t blockPos = symbol.find("VanillaBlockTypeIds::"); + if (blockPos != std::string::npos) { + trackingBlock = symbol.substr(blockPos + 21); + trackingBlockFoundReg = 0; + } + } + } + } + + if (STR_STARTS_WITH4(&buffer[36], "call")) { + std::string_view line(buffer); + auto addressIx = line.find("0x"); + if (addressIx != std::string::npos) { + auto addressStr = line.substr(addressIx + 2); + if (haveSymbolForAddress(addressStr)) { + auto symbol = getSymbolForAddress(addressStr); + if (trackingBlockFoundReg > 0) { + size_t traitPos = symbol.find("BlockTrait::"); + if (traitPos != std::string::npos) { + auto traitStr = symbol.substr(traitPos); + std::cout << "BlockTrait\t" << trackingBlock << "\t" << symbol << std::endl; + } + } + size_t pos = symbol.find("registerBlock<"); + if (pos != std::string::npos) { + trackingBlockFoundReg = trackingBlock.empty() ? 0 : 40; + } + } + } + } + + if (trackingBlockFoundReg) { + trackingBlockFoundReg--; + if (trackingBlockFoundReg == 0) + trackingBlock.clear(); + } + // 140064b3f: 04 00 00 00 if (inMovInstruction && buffer[36] == '\0' && currentHash.size() > 0) { // this instruction is not an instruction but a continuation of the previous mov instruction @@ -469,6 +576,7 @@ void loadDisassembly2(std::string filePath) { } } } + ZeroMemory(buffer, bufferSize); } @@ -513,6 +621,7 @@ int main(int argc, char **argv) { loadDisassembly(disFile); } else if (stage == "-s2") { loadStage1(file); + //loadStage4("stage4.txt"); loadDisassembly2(disFile); } std::cerr << "Done" << std::endl; diff --git a/.github/helper-bot/disa.h b/.github/helper-bot/disa.h index f97295c5..6c9618cd 100644 --- a/.github/helper-bot/disa.h +++ b/.github/helper-bot/disa.h @@ -308,6 +308,8 @@ union RegisterVal { enum RegisterDataType { RDTGeneric, RDTVanillaState, RDTBlockTypeID }; u64 value; char symbolValue[MAX_SYMBOL_SIZE + 1]; + + double doubleValue() { return value == 0 ? -0 : *(double *)&value; } }; struct RegisterState { RegisterVal rax; // @@ -362,10 +364,11 @@ enum Register { void registerClearState() { g_registerState = RegisterState{}; } -#define STR_STARTS_WITH2(str, other) (str[0] == other[0] && str[1] == other[1]) -#define STR_STARTS_WITH3(str, other) (str[0] == other[0] && str[1] == other[1] && str[2] == other[2]) -#define STR_STARTS_WITH4(s, o) (s[0] == o[0] && s[1] == o[1] && s[2] == o[2] && s[3] == o[3]) -#define STR_STARTS_WITH5(s, o) (s[0] == o[0] && s[1] == o[1] && s[2] == o[2] && s[3] == o[3] && s[4] == o[4]) +#define STR_STARTS_WITH2(str, other) ((str)[0] == other[0] && (str)[1] == other[1]) +#define STR_STARTS_WITH3(str, other) ((str)[0] == other[0] && (str)[1] == other[1] && str[2] == other[2]) +#define STR_STARTS_WITH4(s, o) ((s)[0] == o[0] && (s)[1] == o[1] && (s)[2] == o[2] && (s)[3] == o[3]) +#define STR_STARTS_WITH5(s, o) ((s)[0] == o[0] && (s)[1] == o[1] && (s)[2] == o[2] && (s)[3] == o[3] && (s)[4] == o[4]) +#define STR_STARTS_WITH6(s, o) ((s)[0] == o[0] && (s)[1] == o[1] && (s)[2] == o[2] && (s)[3] == o[3] && (s)[4] == o[4] && (s)[5] == o[5]) Register registerGetType(std::string_view str) { // in AT&T syntax, registers are prefixed with % @@ -420,6 +423,11 @@ RegisterVal registerGetArgFloat(int index) { return RegisterVal{}; } } +// double registerGetArgFloatVal(int index) { +// // Registers are stored as u64 so we need to cast to doubles +// u64 value = registerGetArgFloat(index).value; +// return *(double *)&value; +// } RegisterVal registerGetArgInt(int index) { switch (index) { @@ -607,4 +615,24 @@ void registerProcessInstruction(Instruction &instr) { default: break; } +} + +void registerDumpCallArgs() { + auto arg1 = registerGetArgInt(0); + auto arg2 = registerGetArgInt(1); + auto arg3 = registerGetArgInt(2); + auto fArg1 = registerGetArgFloat(0); + auto fArg2 = registerGetArgFloat(1); + auto fArg3 = registerGetArgFloat(2); + // clang-format off + fprintf( + stderr, + "Args: iArg1: %lld (%s), iArg2: %lld (%s), iArg3: %lld (%s) ; fArg1: %f (%s), fArg2: %f (%s), fArg3: %f (%s)\n", + arg1.value, arg1.symbolValue, + arg2.value, arg2.symbolValue, + arg3.value, arg3.symbolValue, + fArg1.doubleValue(), fArg1.symbolValue, + fArg2.doubleValue(), fArg2.symbolValue, + fArg3.doubleValue(), fArg3.symbolValue); + // clang-format on } \ No newline at end of file diff --git a/.github/helper-bot/pdba.cpp b/.github/helper-bot/pdba.cpp index 6c493395..0b085836 100644 --- a/.github/helper-bot/pdba.cpp +++ b/.github/helper-bot/pdba.cpp @@ -1,5 +1,6 @@ #include #include +#include #include void readVanillaState(std::string &demangled) { @@ -36,9 +37,74 @@ void readConstant(std::string &demangled) { std::cout << "SCT\t" << sharedName << "\t" << type << std::endl; } -void loadDump() { +unsigned int parseInt(const std::string &str) { + unsigned int result = 0; + for (auto c : str) { + result = result * 10 + (c - '0'); + } + return result; +} + +std::string int2hex(uint64_t i) { + std::stringstream stream; + stream << std::hex << i; + return stream.str(); +} + +uint64_t hex2int(const std::string &hex) { + uint64_t result = 0; + for (auto c : hex) { + result = result * 16 + (c >= '0' && c <= '9' ? c - '0' : c - 'a' + 10); + } + return result; +} + +int find(std::string &what, std::string subStr) { + for (int i = 0; i < what.size(); i++) { + if (what[i] == subStr[0]) { + bool found = true; + for (int j = 1; j < subStr.size(); j++) { + if (what[i + j] != subStr[j]) { + found = false; + break; + } + } + if (found) { + return i; + } + } + } + return -1; +} + +#define STR_INCLUDES(haystack, needle) (haystack.find(needle) != std::string::npos) + +void loadDump(uint64_t textOffset = 0x140001000, uint64_t relocOffset = 0x142bbd000) { + uint64_t newOffset = relocOffset + 0x4A0000; std::string line; + std::string readingBlockTrait; + while (std::getline(std::cin, line)) { + if (readingBlockTrait.size() > 0) { + // flags = function, addr = 0001:30564736 + auto symInfo = line.find("addr = "); + if (symInfo != std::string::npos) { + auto addr = line.find(":"); + if (addr != std::string::npos) { + auto sectionId = line[addr - 1]; + auto addrStr = line.substr(addr + 1); + if (sectionId == '1') { // .text + auto addrInt = parseInt(addrStr) + textOffset; + std::cout << "WSYM\t" << int2hex(addrInt) << "\t" << readingBlockTrait << std::endl; + } else if (sectionId == '3') { // .data + auto addrInt = parseInt(addrStr) + newOffset; + std::cout << "WSYM\t" << int2hex(addrInt) << "\t" << readingBlockTrait << std::endl; + } + } + } + readingBlockTrait = ""; + } + auto pos = line.find("`?"); if (pos != std::string::npos) { std::string mangledName = line.substr(pos + 1, line.size() - pos - 2); @@ -49,17 +115,31 @@ void loadDump() { } else if (mangledName.find("SharedConstants") != std::string::npos) { readConstant(demangled); } + if (STR_INCLUDES(line, "S_PUB32")) { + // Record only on the interesting things + if (STR_INCLUDES(line, "BlockTrait") || STR_INCLUDES(line, "VanillaBlockTypeIds") || + STR_INCLUDES(line, "registerBlock")) { + readingBlockTrait = demangled; + } + } } } + printf("Done\n"); } -/* -char *microsoftDemangle(const char *mangled_name, size_t *n_read, char *buf, - size_t *n_buf, int *status, - MSDemangleFlags Flags = MSDF_None); -*/ +// 0 .text 022692fc 0000000140001000 0000000140001000 00000400 2**4 -int main() { - loadDump(); +int main(int argc, char **argv) { + if (argc < 3) { + std::cerr << "Usage: " << argv[0] << " " << std::endl; + return 1; + } + std::string textOffsetStr = argv[1]; + std::string relocOffsetStr = argv[2]; + std::cerr << "textOffset: " << textOffsetStr << std::endl; + std::cerr << "relocOffset: " << relocOffsetStr << std::endl; + uint64_t textOffset = hex2int(textOffsetStr); + uint64_t relocOffset = hex2int(relocOffsetStr); + loadDump(textOffset, relocOffset); return 0; } From 71ef1cf2c360b5cd0f97e682ef65c89a3141cb61 Mon Sep 17 00:00:00 2001 From: extremeheat Date: Thu, 11 Apr 2024 20:53:20 -0400 Subject: [PATCH 45/50] add behavior pack to extract block data --- .github/helper-bot/.gitignore | 10 ++- .github/helper-bot/README.txt | 8 ++ .github/helper-bot/serverScript.js | 34 +++++++ .github/helper-bot/update1.js | 139 ++++++++++++++++++++++++++++- .github/helper-bot/update3.js | 3 +- 5 files changed, 187 insertions(+), 7 deletions(-) create mode 100644 .github/helper-bot/README.txt create mode 100644 .github/helper-bot/serverScript.js diff --git a/.github/helper-bot/.gitignore b/.github/helper-bot/.gitignore index 4c8b7d5d..c21a078f 100644 --- a/.github/helper-bot/.gitignore +++ b/.github/helper-bot/.gitignore @@ -1,5 +1,11 @@ +# server bds-* +# built PDB and server disassembly analysis executables *.exe -stage* +# intermediates from pdba/disa rodata.* -strings.* \ No newline at end of file +strings.* +stage*.txt +# update1 +collected*.json +updatedBody.md \ No newline at end of file diff --git a/.github/helper-bot/README.txt b/.github/helper-bot/README.txt new file mode 100644 index 00000000..e6edd670 --- /dev/null +++ b/.github/helper-bot/README.txt @@ -0,0 +1,8 @@ +1. index.js -- ran initially on CRON to check for updates +2. update1.js -- runs bedrock-protocol client against the updated server to collect data from server->client. Also runs a behavior pack to extract block data. +3. disa.exe -- disassembly analysis for Minecraft bedrock edition server binary (combining data from both Linux/Win binaries) + * x86 disassembly for the server software with symbol information is analogus to decompiling the Minecraft Java Edition + and running various extractors on the decompiled code. + * Can be expanded to extract pretty much any desired data from the server software +4. pdba.exe -- analysis of PDB file for Minecraft bedrock edition Windows server binary +5. update3.js -- aggregate and finalize data, send to llm-services for further handling diff --git a/.github/helper-bot/serverScript.js b/.github/helper-bot/serverScript.js new file mode 100644 index 00000000..28c5e682 --- /dev/null +++ b/.github/helper-bot/serverScript.js @@ -0,0 +1,34 @@ +// Minecraft Bedrock Edition behavior pack script to extract block data. +// Based off https://github.com/Alemiz112/BedrockUtils/tree/master/BlockPaletteDumperAddon +import { + BlockStates, + BlockTypes, + BlockPermutation +} from '@minecraft/server' + +const data = { + blocks: [], + blockProperties: BlockStates.getAll() +} + +const blocks = BlockTypes.getAll() +for (let i = 0; i < blocks.length; i++) { + const permutation = BlockPermutation.resolve(blocks[i].id) + const defaultPermutation = permutation.getAllStates() + const blockData = { + name: blocks[i].id, + defaultState: defaultPermutation, + stateTypes: Object.fromEntries(Object.keys(defaultPermutation).map(e => [e, typeof e])), + stateValues: {} + } + const stateNames = Object.keys(defaultPermutation) + for (let j = 0; j < stateNames.length; j++) { + const stateName = stateNames[j] + const state = BlockStates.get(stateName) + const validValues = state.validValues + blockData.stateValues[stateName] = validValues + } + data.blocks.push(blockData) +} + +console.warn('' + JSON.stringify(data) + '') diff --git a/.github/helper-bot/update1.js b/.github/helper-bot/update1.js index 8efcaff7..15ca1110 100644 --- a/.github/helper-bot/update1.js +++ b/.github/helper-bot/update1.js @@ -2,6 +2,7 @@ const fs = require('fs') const path = require('path') const mcData = require('minecraft-data') +const nbt = require('prismarine-nbt') const latestSupportedProtocol = mcData.versions.bedrock[0].version const bedrock = require('bedrock-protocol') const bedrockServer = require('minecraft-bedrock-server') @@ -14,8 +15,8 @@ if (process.env.CI) { } else { globalThis.isMocha = true core = { setOutput: (name, value) => console.log(name, value) } - - github.findIssue = () => ({ body: '(Demo)' }) + github = require('gh-helpers')() + github.getIssue = () => ({ body: '(Demo)' }) } BigInt.prototype.toJSON = function () { @@ -75,6 +76,7 @@ async function main (inputUpdateVer, inputIssueNo) { core.setOutput('serverVersion', serverVersion) core.setOutput('serverPath', serverPath) core.setOutput('serverBin', serverPath + '/bedrock_server_symbols.debug') + fs.rmSync(path.join(serverPath, 'worlds/Bedrock level/world_behavior_packs.json'), { force: true, recursive: true }) const handle = await bedrockServer.startServerAndWait(serverVersion, 60000, { root: __dirname }) await new Promise((resolve) => setTimeout(resolve, 2000)) const pong = await bedrock.ping({ host: '127.0.0.1', port: 19130, timeout: 4000 }) @@ -103,6 +105,20 @@ async function main (inputUpdateVer, inputIssueNo) { core.setOutput('protocolVersion', pong.protocol) } + // If the protocol version was changed, start server again, but with a behavior pack + console.log('⚒️ Re-running Bedrock server with extractor behavior pack') + // First, determine the latest script version + injectPack(path.join(__dirname, 'bds-' + serverVersion)) + const handle2 = await bedrockServer.startServerAndWait(serverVersion, 10000, { root: __dirname }) + const scriptVersion = await collectScriptVersion(handle2) + handle2.kill() + // Re-run the server with the new script version + injectPack(path.join(__dirname, 'bds-' + serverVersion), scriptVersion) + const handle3 = await bedrockServer.startServerAndWait(serverVersion, 10000, { root: __dirname }) + const blockData = await collectDump(handle3) + fs.writeFileSync(path.join(__dirname, '/collectedBlockData.json'), blockData) + handle3.kill() + console.log('✅ Finished working with Linux server binary') console.log('Working now on Windows') const winPath = serverPath.replace('bds-', 'bds-win-') @@ -113,5 +129,120 @@ async function main (inputUpdateVer, inputIssueNo) { console.log('✅ Finished working with Windows server binary') } -main(process.env.UPDATE_VERSION, process.env.ISSUE_NUMBER) -// main('1.20.73', 0) +// main(process.env.UPDATE_VERSION, process.env.ISSUE_NUMBER) +main('1.20.73', 0) + +function collectScriptVersion (handle, timeout = 1000 * 20) { + // The scripting API doesn't support semantic versioning with tilde or caret operators + // so we need to extract the version from the server log + let onceTimer + let onceDone + function onceWithDelay (fn, delay) { + if (onceDone) return + clearTimeout(onceTimer) + onceTimer = setTimeout(() => { + fn() + onceDone = true + }, delay) + } + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + reject(Error('Timeout while waiting for dump')) + }, timeout) + let total = '' + function process (log) { + const data = log.toString() + total += data + for (const line of total.split('\n')) { + if (line.includes('@minecraft/server -')) { + onceWithDelay(() => { + const scriptVersion = line.split('@minecraft/server -')[1].trim() + console.log('Latest @minecraft/server version is', scriptVersion) + clearTimeout(timer) + resolve(scriptVersion) + handle.stdout.off('data', process) + }, 500) + } + } + } + handle.stdout.on('data', process) + }) +} + +function collectDump (handle, timeout = 1000 * 60 * 2) { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + reject(Error('Timeout while waiting for dump')) + }, timeout) + let total = '' + function process (log) { + const data = log.toString() + total += data + for (const line of total.split('\n')) { + if (line.includes('') && line.includes('')) { + console.log('Found dump data!!') + const blockData = line.split('')[1].split('')[0] + clearTimeout(timer) + resolve(blockData) + handle.stdout.off('data', process) + return + } + } + } + handle.stdout.on('data', process) + }) +} + +function injectPack (serverPath, scriptVersion) { + const localScriptPath = path.join(__dirname, 'serverScript.js') + const serverPackPath = path.join(serverPath, 'development_behavior_packs') + const packScriptPath = path.join(serverPackPath, 'extractor/scripts/') + + fs.mkdirSync(packScriptPath, { recursive: true }) + fs.copyFileSync(localScriptPath, packScriptPath + 'main.js') + fs.writeFileSync(path.join(serverPackPath, 'extractor/scripts/main.js'), fs.readFileSync(localScriptPath)) + + packManifest.dependencies[0].version = scriptVersion || '1.0.0-beta' + fs.writeFileSync(path.join(serverPackPath, 'extractor/manifest.json'), JSON.stringify(packManifest, null, 2)) + console.log(JSON.stringify(packManifest, null, 2)) + const levelPath = path.join(serverPath, 'worlds/Bedrock level/level.dat') + const tagBuf = fs.readFileSync(levelPath) + const parsed = nbt.parseUncompressed(tagBuf.slice(8), 'little') + // console.log('Loaded world level.dat', nbt.simplify(parsed)) + parsed.value.experiments = nbt.comp({ + experiments_ever_used: nbt.byte(1), + gametest: nbt.byte(1), + saved_with_toggled_experiments: nbt.byte(1) + }) + const tagHead = Buffer.from([0x0A, 0, 0, 0, 0, 0, 0, 0]) + const tagBody = nbt.writeUncompressed(parsed, 'little') + tagHead.writeUInt32LE(tagBody.length, 4) + fs.writeFileSync(levelPath, Buffer.concat([tagHead, tagBody])) + console.log('Updated world level.dat', levelPath) + const worldPacks = [{ pack_id: 'f604a121-974a-3e04-927a-8a1c9518c96a', version: [1, 0, 0] }] + fs.writeFileSync(path.join(serverPath, 'worlds/Bedrock level/world_behavior_packs.json'), JSON.stringify(worldPacks, null, 2)) + console.log('Updated world behavior_packs.json', worldPacks) +} + +const packManifest = { + format_version: 2, + header: { + allow_random_seed: false, + description: 'DataExtractor', + name: 'DataExtractor', + platform_locked: false, + uuid: 'f604a121-974a-3e04-927a-8a1c9518c96a', + version: [1, 0, 0], + min_engine_version: [1, 20, 0] + }, + modules: [{ + type: 'script', + language: 'javascript', + uuid: 'fa04a121-974a-3e04-927a-8a1c9518c96a', + entry: 'scripts/main.js', + version: [0, 1, 0] + }], + dependencies: [ + { module_name: '@minecraft/server', version: '1.0.0-beta' } + ] +} diff --git a/.github/helper-bot/update3.js b/.github/helper-bot/update3.js index 3f09406f..f4c519e3 100644 --- a/.github/helper-bot/update3.js +++ b/.github/helper-bot/update3.js @@ -13,7 +13,8 @@ async function main () { async function upload () { const artifact = await github.artifacts.createTextArtifact('updateData-' + process.env.UPDATE_VERSION, { extracted: fs.readFileSync('merged.txt', 'latin1'), - collected: JSON.stringify(require('./collected.json')) + collected: JSON.stringify(require('./collected.json')), + collectedBlockData: JSON.stringify(require('./collectedBlockData.json')) }) console.log('Created artifact', artifact) const dispatch = await github.sendWorkflowDispatch({ From c981916b31950d78404f0c5afd8da4b548963c45 Mon Sep 17 00:00:00 2001 From: extremeheat Date: Thu, 11 Apr 2024 21:56:35 -0400 Subject: [PATCH 46/50] update pdba --- .github/helper-bot/disa.cpp | 29 +++++++--------- .github/helper-bot/disa.h | 51 ++++++++++++++--------------- .github/helper-bot/pdba.cpp | 11 ++++--- .github/workflows/update-helper.yml | 5 ++- 4 files changed, 47 insertions(+), 49 deletions(-) diff --git a/.github/helper-bot/disa.cpp b/.github/helper-bot/disa.cpp index c43350d7..5e039ecb 100644 --- a/.github/helper-bot/disa.cpp +++ b/.github/helper-bot/disa.cpp @@ -260,7 +260,6 @@ void loadDisassembly(std::string filePath) { if (pos != std::string::npos) { trackingBlock = line.substr(pos + 21, line.size() - pos - 22); } - } } else { // B1. cont. Sometimes the movabs with hash is not after 2x lea ops, so we dump what we have and continue @@ -377,8 +376,6 @@ void loadDisassembly(std::string filePath) { // STAGE 2 -#define STR_INCLUDES(haystack, needle) (haystack.find(needle) != std::string::npos) - // StateHash -> integer data for this state (like number of variants) std::map> stateVariantMap; @@ -428,10 +425,10 @@ void loadStage1(std::string filePath) { } } } - fprintf(stderr, "Loaded %d state variants from stage1\n", stateVariantMap.size()); + fprintf(stderr, "Loaded %lld state variants from stage1\n", stateVariantMap.size()); } -std::map symbolMap; +std::map symbolMap; void loadStage4(std::string filePath) { // load stage1 which is the output of above loadDisassembly function @@ -452,24 +449,21 @@ void loadStage4(std::string filePath) { std::string name, address, symbolName, _; split4(line, name, address, symbolName, _); if (name == "WSYM") { - symbolMap[address] = symbolName; + uint64_t addressInt = hexStr2Int64(address); + symbolMap[addressInt] = symbolName; } } } - fprintf(stderr, "Loaded %d symbols from stage4\n", symbolMap.size()); + fprintf(stderr, "Loaded %lld symbols from stage4\n", symbolMap.size()); } -bool haveSymbolForAddress(std::string address) { - return symbolMap.find(address) != symbolMap.end(); -} -bool haveSymbolForAddress(std::string_view address) { - return haveSymbolForAddress(std::string(address)); -} -std::string getSymbolForAddress(std::string address) { - return symbolMap[address]; +bool haveSymbolForAddress(std::string_view addressStr) { + uint64_t addr = hexStr2Int64(addressStr); + return symbolMap.find(addr) != symbolMap.end(); } + std::string getSymbolForAddress(std::string_view address) { - return getSymbolForAddress(std::string(address)); + return symbolMap[hexStr2Int64(address)]; } void loadDisassembly2(std::string filePath) { @@ -621,7 +615,8 @@ int main(int argc, char **argv) { loadDisassembly(disFile); } else if (stage == "-s2") { loadStage1(file); - //loadStage4("stage4.txt"); + // Not yet implemented: trait data extraction. This will require re-ordering and running stage4 before doing stage2. + // loadStage4("stage4.txt"); loadDisassembly2(disFile); } std::cerr << "Done" << std::endl; diff --git a/.github/helper-bot/disa.h b/.github/helper-bot/disa.h index 6c9618cd..4ee81e41 100644 --- a/.github/helper-bot/disa.h +++ b/.github/helper-bot/disa.h @@ -1,8 +1,18 @@ #include +#include #include #include #include +// clang-format off +#define STR_STARTS_WITH2(str, other) ((str)[0] == other[0] && (str)[1] == other[1]) +#define STR_STARTS_WITH3(str, other) ((str)[0] == other[0] && (str)[1] == other[1] && str[2] == other[2]) +#define STR_STARTS_WITH4(s, o) ((s)[0] == o[0] && (s)[1] == o[1] && (s)[2] == o[2] && (s)[3] == o[3]) +#define STR_STARTS_WITH5(s, o) ((s)[0] == o[0] && (s)[1] == o[1] && (s)[2] == o[2] && (s)[3] == o[3] && (s)[4] == o[4]) +#define STR_STARTS_WITH6(s, o) ((s)[0] == o[0] && (s)[1] == o[1] && (s)[2] == o[2] && (s)[3] == o[3] && (s)[4] == o[4] && (s)[5] == o[5]) +#define STR_INCLUDES(haystack, needle) (haystack.find(needle) != std::string::npos) +// clang-format on + typedef unsigned long long int u64; void ZeroMemory(char *buffer, int size) { @@ -309,7 +319,9 @@ union RegisterVal { u64 value; char symbolValue[MAX_SYMBOL_SIZE + 1]; - double doubleValue() { return value == 0 ? -0 : *(double *)&value; } + double doubleValue() { + return value == 0 ? -0 : *(double *)&value; + } }; struct RegisterState { RegisterVal rax; // @@ -362,13 +374,9 @@ enum Register { XMM3 }; -void registerClearState() { g_registerState = RegisterState{}; } - -#define STR_STARTS_WITH2(str, other) ((str)[0] == other[0] && (str)[1] == other[1]) -#define STR_STARTS_WITH3(str, other) ((str)[0] == other[0] && (str)[1] == other[1] && str[2] == other[2]) -#define STR_STARTS_WITH4(s, o) ((s)[0] == o[0] && (s)[1] == o[1] && (s)[2] == o[2] && (s)[3] == o[3]) -#define STR_STARTS_WITH5(s, o) ((s)[0] == o[0] && (s)[1] == o[1] && (s)[2] == o[2] && (s)[3] == o[3] && (s)[4] == o[4]) -#define STR_STARTS_WITH6(s, o) ((s)[0] == o[0] && (s)[1] == o[1] && (s)[2] == o[2] && (s)[3] == o[3] && (s)[4] == o[4] && (s)[5] == o[5]) +void registerClearState() { + g_registerState = RegisterState{}; +} Register registerGetType(std::string_view str) { // in AT&T syntax, registers are prefixed with % @@ -423,11 +431,6 @@ RegisterVal registerGetArgFloat(int index) { return RegisterVal{}; } } -// double registerGetArgFloatVal(int index) { -// // Registers are stored as u64 so we need to cast to doubles -// u64 value = registerGetArgFloat(index).value; -// return *(double *)&value; -// } RegisterVal registerGetArgInt(int index) { switch (index) { @@ -568,15 +571,10 @@ void registerProcessInstruction(Instruction &instr) { std::string_view operand1, operand2, operand3; instructionReadOperands(instr, operand1, operand2, operand3); - // std::cout << "Operand1: " << operand1 << std::endl; - // std::cout << "Operand2: " << operand2 << std::endl; - // std::cout << "Operand3: " << operand3 << std::endl; - std::string_view commentSymbol; bool hasCommentSymbol = instr.commentSymbolStart && instr.commentSymbolEnd; if (hasCommentSymbol) { commentSymbol = std::string_view(instr.commentSymbolStart, instr.commentSymbolEnd - instr.commentSymbolStart); - // std::cout << "Reading Comment Symbol: " << commentSymbol << std::endl; } switch (instr.type) { @@ -626,13 +624,14 @@ void registerDumpCallArgs() { auto fArg3 = registerGetArgFloat(2); // clang-format off fprintf( - stderr, - "Args: iArg1: %lld (%s), iArg2: %lld (%s), iArg3: %lld (%s) ; fArg1: %f (%s), fArg2: %f (%s), fArg3: %f (%s)\n", - arg1.value, arg1.symbolValue, - arg2.value, arg2.symbolValue, - arg3.value, arg3.symbolValue, - fArg1.doubleValue(), fArg1.symbolValue, - fArg2.doubleValue(), fArg2.symbolValue, - fArg3.doubleValue(), fArg3.symbolValue); + stderr, + "Args: iArg1: %lld (%s), iArg2: %lld (%s), iArg3: %lld (%s) ; fArg1: %f (%s), fArg2: %f (%s), fArg3: %f (%s)\n", + arg1.value, arg1.symbolValue, + arg2.value, arg2.symbolValue, + arg3.value, arg3.symbolValue, + fArg1.doubleValue(), fArg1.symbolValue, + fArg2.doubleValue(), fArg2.symbolValue, + fArg3.doubleValue(), fArg3.symbolValue + ); // clang-format on } \ No newline at end of file diff --git a/.github/helper-bot/pdba.cpp b/.github/helper-bot/pdba.cpp index 0b085836..a821f5cb 100644 --- a/.github/helper-bot/pdba.cpp +++ b/.github/helper-bot/pdba.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include void readVanillaState(std::string &demangled) { @@ -94,10 +95,10 @@ void loadDump(uint64_t textOffset = 0x140001000, uint64_t relocOffset = 0x142bbd auto sectionId = line[addr - 1]; auto addrStr = line.substr(addr + 1); if (sectionId == '1') { // .text - auto addrInt = parseInt(addrStr) + textOffset; + uint64_t addrInt = parseInt(addrStr) + textOffset; std::cout << "WSYM\t" << int2hex(addrInt) << "\t" << readingBlockTrait << std::endl; } else if (sectionId == '3') { // .data - auto addrInt = parseInt(addrStr) + newOffset; + uint64_t addrInt = parseInt(addrStr) + newOffset; std::cout << "WSYM\t" << int2hex(addrInt) << "\t" << readingBlockTrait << std::endl; } } @@ -135,11 +136,11 @@ int main(int argc, char **argv) { return 1; } std::string textOffsetStr = argv[1]; - std::string relocOffsetStr = argv[2]; + std::string relocOffsetStr = argv[2]; std::cerr << "textOffset: " << textOffsetStr << std::endl; std::cerr << "relocOffset: " << relocOffsetStr << std::endl; - uint64_t textOffset = hex2int(textOffsetStr); - uint64_t relocOffset = hex2int(relocOffsetStr); + uint64_t textOffset = std::stol(textOffsetStr, nullptr, 16); + uint64_t relocOffset = std::stol(relocOffsetStr, nullptr, 16); loadDump(textOffset, relocOffset); return 0; } diff --git a/.github/workflows/update-helper.yml b/.github/workflows/update-helper.yml index ccde34d4..28a8aa0b 100644 --- a/.github/workflows/update-helper.yml +++ b/.github/workflows/update-helper.yml @@ -75,7 +75,10 @@ jobs: run: objdump -d --demangle -M intel ${{ steps.update1Run.outputs.serverWinBin }} | .github/helper-bot/disa.exe -s2 .github/helper-bot/stage1.txt > .github/helper-bot/stage2.txt - name: Resolve PDB symbols (Stage 4) if: steps.update1Run.outputs.needsUpdate - run: llvm-pdbutil-14 dump --all ${{ steps.update1Run.outputs.serverWinPdb }} | .github/helper-bot/pdba.exe > .github/helper-bot/stage4.txt + run: | + WIN_BIN_TEXT_OFF=objdump -h ${{ steps.update1Run.outputs.serverWinBin }} | grep ".text" | cut -b 29-45 + WIN_BIN_RELOC_OFF=objdump -h ${{ steps.update1Run.outputs.serverWinBin }} | grep ".reloc" | cut -b 29-45 + llvm-pdbutil-14 dump --all ${{ steps.update1Run.outputs.serverWinPdb }} | .github/helper-bot/pdba.exe $WIN_BIN_TEXT_OFF $WIN_BIN_RELOC_OFF > .github/helper-bot/stage4.txt # We use Artifacts API for cross repo com which is only avaliable within an Action, but not normal run jobs. # So we need to store the keys in the env for update3 to use to do a workflow dispatch # as in https://github.com/extremeheat/gh-helpers/blob/main/.github/workflows/ci.yml From a77036b82188fcaecf90264a2234481aa572850b Mon Sep 17 00:00:00 2001 From: extremeheat Date: Fri, 12 Apr 2024 08:35:47 -0400 Subject: [PATCH 47/50] update1: move some server utils to minecraft-bedrock-server --- .github/helper-bot/update1.js | 126 ++++++++++++++-------------------- 1 file changed, 53 insertions(+), 73 deletions(-) diff --git a/.github/helper-bot/update1.js b/.github/helper-bot/update1.js index 15ca1110..e436e3f9 100644 --- a/.github/helper-bot/update1.js +++ b/.github/helper-bot/update1.js @@ -2,7 +2,6 @@ const fs = require('fs') const path = require('path') const mcData = require('minecraft-data') -const nbt = require('prismarine-nbt') const latestSupportedProtocol = mcData.versions.bedrock[0].version const bedrock = require('bedrock-protocol') const bedrockServer = require('minecraft-bedrock-server') @@ -32,15 +31,22 @@ async function tryConnect (opts) { }) client.on('connect_allowed', () => { Object.assign(client.options, opts) }) + const collected = {} + const forCollection = ['start_game', 'available_commands'] + return new Promise((resolve, reject) => { - let timeout // eslint-disable-line + const timeout = setTimeout(() => { + if (Object.keys(collected).length !== forCollection.length) { + reject(Error('Unable to collect all packets')) + } + }, 1000 * 60 * 2) + function done (data) { client.close() clearTimeout(timeout) resolve(data) } - const collected = {} - const forCollection = ['start_game', 'available_commands'] + for (const packet of forCollection) { console.log('Waiting for', packet) client.once(packet, (data) => { @@ -52,12 +58,6 @@ async function tryConnect (opts) { } }) } - - timeout = setTimeout(() => { - if (Object.keys(collected).length !== forCollection.length) { - reject(Error('Unable to collect all packets')) - } - }, 1000 * 60 * 2) }) } @@ -70,14 +70,14 @@ async function main (inputUpdateVer, inputIssueNo) { if (serverVersion !== inputUpdateVer) { updatedBody = updatedBody.replace('', `Server version${serverVersion}`) } - const root = __dirname - const serverPath = root + '/bds-' + serverVersion - console.log('Server version', serverVersion, root, 'Server path', serverPath) + const server = await bedrockServer.prepare(serverVersion, { root: __dirname }) + const serverPath = server.path + console.log('Server version', serverVersion, 'Server path', serverPath) core.setOutput('serverVersion', serverVersion) core.setOutput('serverPath', serverPath) core.setOutput('serverBin', serverPath + '/bedrock_server_symbols.debug') - fs.rmSync(path.join(serverPath, 'worlds/Bedrock level/world_behavior_packs.json'), { force: true, recursive: true }) - const handle = await bedrockServer.startServerAndWait(serverVersion, 60000, { root: __dirname }) + await server.clearBehaviorPacks() + const handle = await server.startAndWaitReady(60000) await new Promise((resolve) => setTimeout(resolve, 2000)) const pong = await bedrock.ping({ host: '127.0.0.1', port: 19130, timeout: 4000 }) updatedBody = updatedBody.replace('', `Protocol ID${pong.protocol} (${pong.version})`) @@ -107,14 +107,16 @@ async function main (inputUpdateVer, inputIssueNo) { // If the protocol version was changed, start server again, but with a behavior pack console.log('⚒️ Re-running Bedrock server with extractor behavior pack') + // First, determine the latest script version - injectPack(path.join(__dirname, 'bds-' + serverVersion)) - const handle2 = await bedrockServer.startServerAndWait(serverVersion, 10000, { root: __dirname }) + injectPack2(server, '1.0.0-beta') + const handle2 = await server.startAndWaitReady(10000) const scriptVersion = await collectScriptVersion(handle2) handle2.kill() + // Re-run the server with the new script version - injectPack(path.join(__dirname, 'bds-' + serverVersion), scriptVersion) - const handle3 = await bedrockServer.startServerAndWait(serverVersion, 10000, { root: __dirname }) + injectPack2(server, scriptVersion) + const handle3 = await server.startAndWaitReady(10000) const blockData = await collectDump(handle3) fs.writeFileSync(path.join(__dirname, '/collectedBlockData.json'), blockData) handle3.kill() @@ -129,8 +131,8 @@ async function main (inputUpdateVer, inputIssueNo) { console.log('✅ Finished working with Windows server binary') } -// main(process.env.UPDATE_VERSION, process.env.ISSUE_NUMBER) -main('1.20.73', 0) +main(process.env.UPDATE_VERSION, process.env.ISSUE_NUMBER) +// if (!process.env.CI) main('1.20.73', 0) function collectScriptVersion (handle, timeout = 1000 * 20) { // The scripting API doesn't support semantic versioning with tilde or caret operators @@ -193,56 +195,34 @@ function collectDump (handle, timeout = 1000 * 60 * 2) { }) } -function injectPack (serverPath, scriptVersion) { - const localScriptPath = path.join(__dirname, 'serverScript.js') - const serverPackPath = path.join(serverPath, 'development_behavior_packs') - const packScriptPath = path.join(serverPackPath, 'extractor/scripts/') - - fs.mkdirSync(packScriptPath, { recursive: true }) - fs.copyFileSync(localScriptPath, packScriptPath + 'main.js') - fs.writeFileSync(path.join(serverPackPath, 'extractor/scripts/main.js'), fs.readFileSync(localScriptPath)) - - packManifest.dependencies[0].version = scriptVersion || '1.0.0-beta' - fs.writeFileSync(path.join(serverPackPath, 'extractor/manifest.json'), JSON.stringify(packManifest, null, 2)) - console.log(JSON.stringify(packManifest, null, 2)) - const levelPath = path.join(serverPath, 'worlds/Bedrock level/level.dat') - const tagBuf = fs.readFileSync(levelPath) - const parsed = nbt.parseUncompressed(tagBuf.slice(8), 'little') - // console.log('Loaded world level.dat', nbt.simplify(parsed)) - parsed.value.experiments = nbt.comp({ - experiments_ever_used: nbt.byte(1), - gametest: nbt.byte(1), - saved_with_toggled_experiments: nbt.byte(1) - }) - const tagHead = Buffer.from([0x0A, 0, 0, 0, 0, 0, 0, 0]) - const tagBody = nbt.writeUncompressed(parsed, 'little') - tagHead.writeUInt32LE(tagBody.length, 4) - fs.writeFileSync(levelPath, Buffer.concat([tagHead, tagBody])) - console.log('Updated world level.dat', levelPath) - const worldPacks = [{ pack_id: 'f604a121-974a-3e04-927a-8a1c9518c96a', version: [1, 0, 0] }] - fs.writeFileSync(path.join(serverPath, 'worlds/Bedrock level/world_behavior_packs.json'), JSON.stringify(worldPacks, null, 2)) - console.log('Updated world behavior_packs.json', worldPacks) -} - -const packManifest = { - format_version: 2, - header: { - allow_random_seed: false, - description: 'DataExtractor', - name: 'DataExtractor', - platform_locked: false, - uuid: 'f604a121-974a-3e04-927a-8a1c9518c96a', - version: [1, 0, 0], - min_engine_version: [1, 20, 0] - }, - modules: [{ - type: 'script', - language: 'javascript', - uuid: 'fa04a121-974a-3e04-927a-8a1c9518c96a', - entry: 'scripts/main.js', - version: [0, 1, 0] - }], - dependencies: [ - { module_name: '@minecraft/server', version: '1.0.0-beta' } - ] +function injectPack2 (/** @type {import('minecraft-bedrock-server').BedrockVanillaServer} */ server, scriptVersion) { + server.clearBehaviorPacks() + server.addQuickScript({ + manifest: { + format_version: 2, + header: { + allow_random_seed: false, + description: 'DataExtractor', + name: 'DataExtractor', + platform_locked: false, + uuid: 'f604a121-974a-3e04-927a-8a1c9518c96a', + version: [1, 0, 0], + min_engine_version: [1, 20, 0] + }, + modules: [{ + type: 'script', + language: 'javascript', + uuid: 'fa04a121-974a-3e04-927a-8a1c9518c96a', + entry: 'scripts/main.js', + version: [0, 1, 0] + }], + dependencies: [ + { module_name: '@minecraft/server', version: scriptVersion || '1.0.0-beta' } + ] + }, + scripts: { + 'scripts/main.js': path.join(__dirname, 'serverScript.js') + } + }, true, true) + server.toggleExperiments({ gametest: true }) } From 03405b4bcec23065de30f1e0f8ea3b507c673d0d Mon Sep 17 00:00:00 2001 From: extremeheat Date: Fri, 12 Apr 2024 10:46:38 -0400 Subject: [PATCH 48/50] fix workflow command --- .github/helper-bot/update1.js | 7 +++---- .github/workflows/update-helper.yml | 5 ++--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/helper-bot/update1.js b/.github/helper-bot/update1.js index e436e3f9..82e5f532 100644 --- a/.github/helper-bot/update1.js +++ b/.github/helper-bot/update1.js @@ -109,13 +109,13 @@ async function main (inputUpdateVer, inputIssueNo) { console.log('⚒️ Re-running Bedrock server with extractor behavior pack') // First, determine the latest script version - injectPack2(server, '1.0.0-beta') + injectPack(server, '1.0.0-beta') const handle2 = await server.startAndWaitReady(10000) const scriptVersion = await collectScriptVersion(handle2) handle2.kill() // Re-run the server with the new script version - injectPack2(server, scriptVersion) + injectPack(server, scriptVersion) const handle3 = await server.startAndWaitReady(10000) const blockData = await collectDump(handle3) fs.writeFileSync(path.join(__dirname, '/collectedBlockData.json'), blockData) @@ -182,7 +182,6 @@ function collectDump (handle, timeout = 1000 * 60 * 2) { total += data for (const line of total.split('\n')) { if (line.includes('') && line.includes('')) { - console.log('Found dump data!!') const blockData = line.split('')[1].split('')[0] clearTimeout(timer) resolve(blockData) @@ -195,7 +194,7 @@ function collectDump (handle, timeout = 1000 * 60 * 2) { }) } -function injectPack2 (/** @type {import('minecraft-bedrock-server').BedrockVanillaServer} */ server, scriptVersion) { +function injectPack (/** @type {import('minecraft-bedrock-server').BedrockVanillaServer} */ server, scriptVersion) { server.clearBehaviorPacks() server.addQuickScript({ manifest: { diff --git a/.github/workflows/update-helper.yml b/.github/workflows/update-helper.yml index 28a8aa0b..8c2f6c01 100644 --- a/.github/workflows/update-helper.yml +++ b/.github/workflows/update-helper.yml @@ -28,7 +28,6 @@ jobs: working-directory: .github/helper-bot env: GITHUB_TOKEN: ${{ secrets.PAT_PASSWORD }} - # Above step sets output of updateVersion, issueNumber with actions/core update: name: Update @@ -76,8 +75,8 @@ jobs: - name: Resolve PDB symbols (Stage 4) if: steps.update1Run.outputs.needsUpdate run: | - WIN_BIN_TEXT_OFF=objdump -h ${{ steps.update1Run.outputs.serverWinBin }} | grep ".text" | cut -b 29-45 - WIN_BIN_RELOC_OFF=objdump -h ${{ steps.update1Run.outputs.serverWinBin }} | grep ".reloc" | cut -b 29-45 + WIN_BIN_TEXT_OFF=`objdump -h ${{ steps.update1Run.outputs.serverWinBin }} | grep ".text" | cut -b 29-45` + WIN_BIN_RELOC_OFF=`objdump -h ${{ steps.update1Run.outputs.serverWinBin }} | grep ".reloc" | cut -b 29-45` llvm-pdbutil-14 dump --all ${{ steps.update1Run.outputs.serverWinPdb }} | .github/helper-bot/pdba.exe $WIN_BIN_TEXT_OFF $WIN_BIN_RELOC_OFF > .github/helper-bot/stage4.txt # We use Artifacts API for cross repo com which is only avaliable within an Action, but not normal run jobs. # So we need to store the keys in the env for update3 to use to do a workflow dispatch From ca6f9b466673a197e4c693f185067cfac31a59af Mon Sep 17 00:00:00 2001 From: extremeheat Date: Fri, 12 Apr 2024 12:23:34 -0400 Subject: [PATCH 49/50] remove some debug --- .github/helper-bot/disa.cpp | 4 ++-- .github/helper-bot/index.js | 18 ++++++++---------- .github/helper-bot/update1.js | 6 +++--- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/.github/helper-bot/disa.cpp b/.github/helper-bot/disa.cpp index 5e039ecb..975ceca9 100644 --- a/.github/helper-bot/disa.cpp +++ b/.github/helper-bot/disa.cpp @@ -425,7 +425,7 @@ void loadStage1(std::string filePath) { } } } - fprintf(stderr, "Loaded %lld state variants from stage1\n", stateVariantMap.size()); + fprintf(stderr, "Loaded %lu state variants from stage1\n", stateVariantMap.size()); } std::map symbolMap; @@ -454,7 +454,7 @@ void loadStage4(std::string filePath) { } } } - fprintf(stderr, "Loaded %lld symbols from stage4\n", symbolMap.size()); + fprintf(stderr, "Loaded %lu symbols from stage4\n", symbolMap.size()); } bool haveSymbolForAddress(std::string_view addressStr) { diff --git a/.github/helper-bot/index.js b/.github/helper-bot/index.js index 766e8872..152f82e3 100644 --- a/.github/helper-bot/index.js +++ b/.github/helper-bot/index.js @@ -46,8 +46,7 @@ ${commitData} -*I'll try to close this issue automatically if the protocol version didn't change. -If the protocol version did change, the automatic update system will try to complete an update to minecraft-data and bedrock-protocol and if successful it will auto close this issue.* +*I'll try to close this issue automatically if the protocol version didn't change. If the protocol version did change, the automatic update system will try to complete an update to minecraft-data and bedrock-protocol and if successful it will auto close this issue.* ----- @@ -56,11 +55,10 @@ If the protocol version did change, the automatic update system will try to comp } } -function getCommitsInRepo (repo, containing, since) { +async function getCommitsInRepo (repo, containing, since) { const endpoint = `https://api.github.com/repos/${repo}/commits` console.log('Getting', endpoint) - cp.execSync(`curl -L ${endpoint} -o commits.json`, { stdio: 'inherit', shell: true }) - const commits = JSON.parse(fs.readFileSync('./commits.json', 'utf-8')) + const commits = await fetch(endpoint).then(res => res.json()) const relevant = [] for (const commit of commits) { if (commit.commit.message.includes(containing)) { @@ -81,8 +79,8 @@ function getCommitsInRepo (repo, containing, since) { } async function fetchLatest () { - cp.execSync(`curl -L "${latestVersionEndpoint}" -o results.json`, { stdio: 'inherit', shell: true }) - const json = require('./results.json') + // curl -L "https://itunes.apple.com/lookup?bundleId=com.mojang.minecraftpe" -o results.json + const json = await fetch(latestVersionEndpoint).then(res => res.json()) const result = json.results[0] const currentTypes = fs.readFileSync(join(__dirname, '../../index.d.ts'), 'utf-8') @@ -111,9 +109,9 @@ async function fetchLatest () { version = version.replace('.0', '') const issuePayload = buildFirstIssue(title, result, { - PocketMine: getCommitsInRepo('pmmp/PocketMine-MP', version, currentVersionReleaseDate), - gophertunnel: getCommitsInRepo('Sandertv/gophertunnel', version, currentVersionReleaseDate), - CloudburstMC: getCommitsInRepo('CloudburstMC/Protocol', version, currentVersionReleaseDate) + PocketMine: await getCommitsInRepo('pmmp/PocketMine-MP', version, currentVersionReleaseDate), + gophertunnel: await getCommitsInRepo('Sandertv/gophertunnel', version, currentVersionReleaseDate), + CloudburstMC: await getCommitsInRepo('CloudburstMC/Protocol', version, currentVersionReleaseDate) }) if (issueStatus.isOpen) { diff --git a/.github/helper-bot/update1.js b/.github/helper-bot/update1.js index 82e5f532..0ce87ba7 100644 --- a/.github/helper-bot/update1.js +++ b/.github/helper-bot/update1.js @@ -83,10 +83,10 @@ async function main (inputUpdateVer, inputIssueNo) { updatedBody = updatedBody.replace('', `Protocol ID${pong.protocol} (${pong.version})`) try { await tryConnect({ protocolVersion: pong.protocol }) - updatedBody = updatedBody.replace('', 'Partly Already CompatibleYes') + updatedBody = updatedBody.replace('', 'Partly Already CompatibleYes') } catch (e) { console.error(e) - updatedBody = updatedBody.replace('', 'Partly Already CompatibleNO') + updatedBody = updatedBody.replace('', 'Partly Already CompatibleNO') } fs.writeFileSync(path.join(__dirname, '/updatedBody.md'), updatedBody) await github.updateIssue(inputIssueNo, { body: updatedBody }) @@ -94,7 +94,7 @@ async function main (inputUpdateVer, inputIssueNo) { handle.kill() // Check if protocol version has changed - if (false && pong.protocol === latestSupportedProtocol) { + if (pong.protocol === latestSupportedProtocol) { console.log('Protocol version has not changed') // Close the github issue await github.close(inputIssueNo, 'Protocol version has not changed, assuming no compatibility issues.') From 8f61ae287caf2ea0cca60f77cc67e83d93893abb Mon Sep 17 00:00:00 2001 From: extremeheat Date: Fri, 19 Apr 2024 00:23:54 -0400 Subject: [PATCH 50/50] Update update3.js --- .github/helper-bot/update3.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/helper-bot/update3.js b/.github/helper-bot/update3.js index f4c519e3..0a42d4b9 100644 --- a/.github/helper-bot/update3.js +++ b/.github/helper-bot/update3.js @@ -24,7 +24,7 @@ async function upload () { inputs: { action: 'minecraft/bedrockDataUpdate', payload: JSON.stringify({ - repoData: await github.getRepoDetails(), + repo: await github.getRepoDetails(), artifactId: artifact.id, artifactSize: artifact.size, updateVersion: process.env.UPDATE_VERSION,