{"payload":{"feedbackUrl":"https://github.com/orgs/community/discussions/53140","repo":{"id":686099738,"defaultBranch":"main","name":"automindx","ownerLogin":"pythaiml","currentUserCanPush":false,"isFork":false,"isEmpty":false,"createdAt":"2023-09-01T18:36:47.000Z","ownerAvatar":"https://avatars.githubusercontent.com/u/141391879?v=4","public":true,"private":false,"isOrgOwned":true},"refInfo":{"name":"","listCacheKey":"v0:1693593516.0","currentOid":""},"activityList":{"items":[{"before":"be32b85fe5615de96bd116edc0e55093f65ce558","after":"3872562b30d15fa7617264981064123623a27e44","ref":"refs/heads/main","pushedAt":"2024-04-24T18:46:22.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"Professor-Codephreak","name":"codephreak","path":"/Professor-Codephreak","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/140855987?s=80&v=4"},"commit":{"message":"Update INSTALL.md\n\nhttps://github.com/pythaiml/automindx/blob/main/automindx.install<br />\r\n\r\neasy automindx.install\r\ndownload automindx.install<br <br />>\r\nchmod +x automindx.install && ./automindx.install<br />","shortMessageHtmlLink":"Update INSTALL.md"}},{"before":"bb4339bef13da7794dfd001dadce5b173125eaef","after":"be32b85fe5615de96bd116edc0e55093f65ce558","ref":"refs/heads/main","pushedAt":"2024-04-24T18:44:12.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"Professor-Codephreak","name":"codephreak","path":"/Professor-Codephreak","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/140855987?s=80&v=4"},"commit":{"message":"Update INSTALL.md\n\nhttps://github.com/pythaiml/automindx/blob/main/automindx.install<br />\r\n\r\neasy automindx.install\r\ndownload automindx.install<br <br />>\r\nchmod +x automindx.install && ./automindx.install<br />","shortMessageHtmlLink":"Update INSTALL.md"}},{"before":"4ce57339831983ad17248f2470c480bbbd2646b9","after":"bb4339bef13da7794dfd001dadce5b173125eaef","ref":"refs/heads/main","pushedAt":"2024-04-24T18:43:46.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"Professor-Codephreak","name":"codephreak","path":"/Professor-Codephreak","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/140855987?s=80&v=4"},"commit":{"message":"Update INSTALL.md\n\nhttps://github.com/pythaiml/automindx/blob/main/automindx.install<br />\r\n\r\neasy automindx.install\r\ndownload automindx.install<br <br />>\r\nchmod +x automindx.install && ./automindx.install<br />","shortMessageHtmlLink":"Update INSTALL.md"}},{"before":"f72884a5b7382e1baf327cd62ee30a6b68d4b169","after":"4ce57339831983ad17248f2470c480bbbd2646b9","ref":"refs/heads/main","pushedAt":"2024-04-23T22:23:30.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"Professor-Codephreak","name":"codephreak","path":"/Professor-Codephreak","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/140855987?s=80&v=4"},"commit":{"message":"Update automindx.install\n\n#!/usr/bin/env bash\r\n\r\n# Script Description:\r\n# This script automates the setup and installation of the automindx environment, including Miniconda, required packages, and the automindx repository.\r\n# Tested on Ubuntu 22.04.6 LTS, Linux Mint 21.2, and Manjaro Linux\r\n# model experiences an error on inputs larger than 4096 characters 4096chunk is not working\r\n# creates a local language model running Professor Codephreak with a desire to create automindx in a conda environment with the default Gradio interface\r\n# Professor Codephreak successfully saves memory.py as local context and parses from aGLM.py\r\n\r\n# Copyright Information:\r\n# codephreak (c) codephreak MIT licence 2023\r\n# automindx (c) codephreak 2023 BSD licence\r\n# mastermind (c) codephreak GPLv3 2024\r\n# aGLM (c) codephreak Apache licence 2024\r\n\r\n# Ensure the script is run with sudo privileges\r\nif [ \"$(id -u)\" != \"0\" ]; then\r\n    echo \"This script must be run as root\" 1>&2\r\n    exit 1\r\nfi\r\n\r\n# Detect user's shell and set shell initialization file\r\nUSER_SHELL=$(echo $SHELL | awk -F'/' '{print $NF}')  # More robust way to determine the shell\r\nINIT_FILE=\".$USER_SHELL\"rc  # Assuming all initialization files follow this pattern\r\n\r\n# Update PATH in user's shell initialization file\r\necho 'export PATH=\"$HOME/.local/bin:$PATH\"' >> \"$HOME/$INIT_FILE\"\r\nsource \"$HOME/$INIT_FILE\"\r\n\r\n# Prompt user to upgrade the system\r\nread -p \"Would you like to upgrade your apt-based system? (y/n): \" UPGRADE_SYSTEM\r\nif [[ $UPGRADE_SYSTEM =~ ^[Yy]$ ]]; then\r\n    echo \"Upgrading system...\"\r\n    apt-get update && apt-get upgrade -y\r\n    echo \"System upgrade completed.\"\r\nelse\r\n    echo \"Skipping system upgrade.\"\r\nfi\r\n\r\n# Check and install Miniconda\r\nif command -v conda &> /dev/null; then\r\n    echo \"Miniconda is already installed.\"\r\nelse\r\n    echo \"Installing Miniconda...\"\r\n    wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh\r\n    bash miniconda.sh -b -p \"$HOME/miniconda3\"\r\n    rm miniconda.sh\r\n    echo 'export PATH=\"$HOME/miniconda3/bin:$PATH\"' >> \"$HOME/$INIT_FILE\"\r\n    source \"$HOME/$INIT_FILE\"\r\n    conda init \"$USER_SHELL\"\r\nfi\r\n\r\n# Initialize Miniconda for the user's shell\r\nconda init \"$USER_SHELL\"\r\nsource \"$HOME/$INIT_FILE\"\r\n\r\n# Setup automindx Miniconda environment\r\nconda create --name automindx python=3.11.4 -y\r\nconda activate automindx\r\n\r\n# Install required packages using pip\r\npip install gradio==3.37.0 psutil==5.9.5 llama2_wrapper==0.1.7 ujson==5.8.0 fire==0.5.0\r\n\r\n# Clone the automindx repository and install its requirements\r\necho \"Cloning the automindx repository...\"\r\ngit clone https://github.com/GATERAGE/aglm/\r\ncd aglm\r\npip install -r requirements.txt\r\n\r\n# Execute the UIUX script with parameters\r\necho \"Running uiux.py...\"\r\npython3 uiux.py --model_name=\"TheBloke/llama2-7b-chat-codeCherryPop-qLoRA-GGML\" --tokenizer_name=\"TheBloke/llama2-7b-chat-codeCherryPop-qLoRA-GGML\" --model_type=\"ggml\" --save_history --file_name=\"llama-2-7b-chat-codeCherryPop.ggmlv3.q4_1.bin\"\r\n\r\necho \"Installation and setup completed successfully.\"","shortMessageHtmlLink":"Update automindx.install"}},{"before":"50ce1968f77ce6bcf4a89650bbee2cc8850f67a7","after":"f72884a5b7382e1baf327cd62ee30a6b68d4b169","ref":"refs/heads/main","pushedAt":"2024-04-23T22:21:14.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"Professor-Codephreak","name":"codephreak","path":"/Professor-Codephreak","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/140855987?s=80&v=4"},"commit":{"message":"Update automindx.install\n\n#!/usr/bin/env bash\r\n\r\n# Script Description:\r\n# This script automates the setup and installation of the automindx environment, including Miniconda, required packages, and the automindx repository.\r\n# Tested on Ubuntu 22.04.6 LTS, Linux Mint 21.2, and Manjaro Linux\r\n# model experiences an error on inputs larger than 4096 characters 4096chunk is not working\r\n# creates a local language model running Professor Codephreak with a desire to create automindx in a conda environment with the default Gradio interface\r\n\r\n# Copyright Information:\r\n# codephreak (c) codephreak MIT licence 2023\r\n# automindx (c) codephreak 2023 BSD licence\r\n# mastermind (c) codephreak GPLv3 2024\r\n# aGLM (c) codephreak Apache licence 2024\r\n\r\n# Ensure the script is run with sudo privileges\r\nif [ \"$(id -u)\" != \"0\" ]; then\r\n    echo \"This script must be run as root\" 1>&2\r\n    exit 1\r\nfi\r\n\r\n# Detect user's shell and set shell initialization file\r\nUSER_SHELL=$(echo $SHELL | awk -F'/' '{print $NF}')  # More robust way to determine the shell\r\nINIT_FILE=\".$USER_SHELL\"rc  # Assuming all initialization files follow this pattern\r\n\r\n# Update PATH in user's shell initialization file\r\necho 'export PATH=\"$HOME/.local/bin:$PATH\"' >> \"$HOME/$INIT_FILE\"\r\nsource \"$HOME/$INIT_FILE\"\r\n\r\n# Prompt user to upgrade the system\r\nread -p \"Would you like to upgrade your apt-based system? (y/n): \" UPGRADE_SYSTEM\r\nif [[ $UPGRADE_SYSTEM =~ ^[Yy]$ ]]; then\r\n    echo \"Upgrading system...\"\r\n    apt-get update && apt-get upgrade -y\r\n    echo \"System upgrade completed.\"\r\nelse\r\n    echo \"Skipping system upgrade.\"\r\nfi\r\n\r\n# Check and install Miniconda\r\nif command -v conda &> /dev/null; then\r\n    echo \"Miniconda is already installed.\"\r\nelse\r\n    echo \"Installing Miniconda...\"\r\n    wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh\r\n    bash miniconda.sh -b -p \"$HOME/miniconda3\"\r\n    rm miniconda.sh\r\n    echo 'export PATH=\"$HOME/miniconda3/bin:$PATH\"' >> \"$HOME/$INIT_FILE\"\r\n    source \"$HOME/$INIT_FILE\"\r\n    conda init \"$USER_SHELL\"\r\nfi\r\n\r\n# Initialize Miniconda for the user's shell\r\nconda init \"$USER_SHELL\"\r\nsource \"$HOME/$INIT_FILE\"\r\n\r\n# Setup automindx Miniconda environment\r\nconda create --name automindx python=3.11.4 -y\r\nconda activate automindx\r\n\r\n# Install required packages using pip\r\npip install gradio==3.37.0 psutil==5.9.5 llama2_wrapper==0.1.7 ujson==5.8.0 fire==0.5.0\r\n\r\n# Clone the automindx repository and install its requirements\r\necho \"Cloning the automindx repository...\"\r\ngit clone https://github.com/GATERAGE/aglm/\r\ncd aglm\r\npip install -r requirements.txt\r\n\r\n# Execute the UIUX script with parameters\r\necho \"Running uiux.py...\"\r\npython3 uiux.py --model_name=\"TheBloke/llama2-7b-chat-codeCherryPop-qLoRA-GGML\" --tokenizer_name=\"TheBloke/llama2-7b-chat-codeCherryPop-qLoRA-GGML\" --model_type=\"ggml\" --save_history --file_name=\"llama-2-7b-chat-codeCherryPop.ggmlv3.q4_1.bin\"\r\n\r\necho \"Installation and setup completed successfully.\"","shortMessageHtmlLink":"Update automindx.install"}},{"before":"6a5aa77eee023dfd54cb419c6f7975ecd8818791","after":"50ce1968f77ce6bcf4a89650bbee2cc8850f67a7","ref":"refs/heads/main","pushedAt":"2024-04-23T22:18:52.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"Professor-Codephreak","name":"codephreak","path":"/Professor-Codephreak","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/140855987?s=80&v=4"},"commit":{"message":"Update uiux.py\n\n#!/usr/bin/env bash\r\n\r\n# Script Description:\r\n# This script automates the setup and installation of the automindx environment, including Miniconda, required packages, and the automindx repository.\r\n# Tested on Ubuntu 22.04.6 LTS, Linux Mint 21.2, and Manjaro Linux\r\n# creates a local language model running Professor Codephreak with a desire to create automindx in a conda environment with the default Gradio interface\r\n\r\n# Copyright Information:\r\n# codephreak (c) codephreak MIT licence 2023\r\n# automindx (c) codephreak 2023 BSD licence\r\n# mastermind (c) codephreak GPLv3 2024\r\n# aGLM (c) codephreak Apache licence 2024\r\n\r\n# Ensure the script is run with sudo privileges\r\nif [ \"$(id -u)\" != \"0\" ]; then\r\n    echo \"This script must be run as root\" 1>&2\r\n    exit 1\r\nfi\r\n\r\n# Detect user's shell and set shell initialization file\r\nUSER_SHELL=$(echo $SHELL | awk -F'/' '{print $NF}')  # More robust way to determine the shell\r\nINIT_FILE=\".$USER_SHELL\"rc  # Assuming all initialization files follow this pattern\r\n\r\n# Update PATH in user's shell initialization file\r\necho 'export PATH=\"$HOME/.local/bin:$PATH\"' >> \"$HOME/$INIT_FILE\"\r\nsource \"$HOME/$INIT_FILE\"\r\n\r\n# Prompt user to upgrade the system\r\nread -p \"Would you like to upgrade your apt-based system? (y/n): \" UPGRADE_SYSTEM\r\nif [[ $UPGRADE_SYSTEM =~ ^[Yy]$ ]]; then\r\n    echo \"Upgrading system...\"\r\n    apt-get update && apt-get upgrade -y\r\n    echo \"System upgrade completed.\"\r\nelse\r\n    echo \"Skipping system upgrade.\"\r\nfi\r\n\r\n# Check and install Miniconda\r\nif command -v conda &> /dev/null; then\r\n    echo \"Miniconda is already installed.\"\r\nelse\r\n    echo \"Installing Miniconda...\"\r\n    wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh\r\n    bash miniconda.sh -b -p \"$HOME/miniconda3\"\r\n    rm miniconda.sh\r\n    echo 'export PATH=\"$HOME/miniconda3/bin:$PATH\"' >> \"$HOME/$INIT_FILE\"\r\n    source \"$HOME/$INIT_FILE\"\r\n    conda init \"$USER_SHELL\"\r\nfi\r\n\r\n# Initialize Miniconda for the user's shell\r\nconda init \"$USER_SHELL\"\r\nsource \"$HOME/$INIT_FILE\"\r\n\r\n# Setup automindx Miniconda environment\r\nconda create --name automindx python=3.11.4 -y\r\nconda activate automindx\r\n\r\n# Install required packages using pip\r\npip install gradio==3.37.0 psutil==5.9.5 llama2_wrapper==0.1.7 ujson==5.8.0 fire==0.5.0\r\n\r\n# Clone the automindx repository and install its requirements\r\necho \"Cloning the automindx repository...\"\r\ngit clone https://github.com/GATERAGE/aglm/\r\ncd aglm\r\npip install -r requirements.txt\r\n\r\n# Execute the UIUX script with parameters\r\necho \"Running uiux.py...\"\r\npython3 uiux.py --model_name=\"TheBloke/llama2-7b-chat-codeCherryPop-qLoRA-GGML\" --tokenizer_name=\"TheBloke/llama2-7b-chat-codeCherryPop-qLoRA-GGML\" --model_type=\"ggml\" --save_history --file_name=\"llama-2-7b-chat-codeCherryPop.ggmlv3.q4_1.bin\"\r\n\r\necho \"Installation and setup completed successfully.\"","shortMessageHtmlLink":"Update uiux.py"}},{"before":"749f2913f3e9a391e79732207443abeb68a02b81","after":"6a5aa77eee023dfd54cb419c6f7975ecd8818791","ref":"refs/heads/main","pushedAt":"2024-04-23T22:18:22.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"Professor-Codephreak","name":"codephreak","path":"/Professor-Codephreak","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/140855987?s=80&v=4"},"commit":{"message":"Update automindx.install\n\n#!/usr/bin/env bash\r\n\r\n# Script Description:\r\n# This script automates the setup and installation of the automindx environment, including Miniconda, required packages, and the automindx repository.\r\n# Tested on Ubuntu 22.04.6 LTS, Linux Mint 21.2, and Manjaro Linux\r\n# creates a local language model running Professor Codephreak with a desire to create automindx in a conda environment with the default Gradio interface\r\n\r\n# Copyright Information:\r\n# codephreak (c) codephreak MIT licence 2023\r\n# automindx (c) codephreak 2023 BSD licence\r\n# mastermind (c) codephreak GPLv3 2024\r\n# aGLM (c) codephreak Apache licence 2024\r\n\r\n# Ensure the script is run with sudo privileges\r\nif [ \"$(id -u)\" != \"0\" ]; then\r\n    echo \"This script must be run as root\" 1>&2\r\n    exit 1\r\nfi\r\n\r\n# Detect user's shell and set shell initialization file\r\nUSER_SHELL=$(echo $SHELL | awk -F'/' '{print $NF}')  # More robust way to determine the shell\r\nINIT_FILE=\".$USER_SHELL\"rc  # Assuming all initialization files follow this pattern\r\n\r\n# Update PATH in user's shell initialization file\r\necho 'export PATH=\"$HOME/.local/bin:$PATH\"' >> \"$HOME/$INIT_FILE\"\r\nsource \"$HOME/$INIT_FILE\"\r\n\r\n# Prompt user to upgrade the system\r\nread -p \"Would you like to upgrade your apt-based system? (y/n): \" UPGRADE_SYSTEM\r\nif [[ $UPGRADE_SYSTEM =~ ^[Yy]$ ]]; then\r\n    echo \"Upgrading system...\"\r\n    apt-get update && apt-get upgrade -y\r\n    echo \"System upgrade completed.\"\r\nelse\r\n    echo \"Skipping system upgrade.\"\r\nfi\r\n\r\n# Check and install Miniconda\r\nif command -v conda &> /dev/null; then\r\n    echo \"Miniconda is already installed.\"\r\nelse\r\n    echo \"Installing Miniconda...\"\r\n    wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh\r\n    bash miniconda.sh -b -p \"$HOME/miniconda3\"\r\n    rm miniconda.sh\r\n    echo 'export PATH=\"$HOME/miniconda3/bin:$PATH\"' >> \"$HOME/$INIT_FILE\"\r\n    source \"$HOME/$INIT_FILE\"\r\n    conda init \"$USER_SHELL\"\r\nfi\r\n\r\n# Initialize Miniconda for the user's shell\r\nconda init \"$USER_SHELL\"\r\nsource \"$HOME/$INIT_FILE\"\r\n\r\n# Setup automindx Miniconda environment\r\nconda create --name automindx python=3.11.4 -y\r\nconda activate automindx\r\n\r\n# Install required packages using pip\r\npip install gradio==3.37.0 psutil==5.9.5 llama2_wrapper==0.1.7 ujson==5.8.0 fire==0.5.0\r\n\r\n# Clone the automindx repository and install its requirements\r\necho \"Cloning the automindx repository...\"\r\ngit clone https://github.com/GATERAGE/aglm/\r\ncd aglm\r\npip install -r requirements.txt\r\n\r\n# Execute the UIUX script with parameters\r\necho \"Running uiux.py...\"\r\npython3 uiux.py --model_name=\"TheBloke/llama2-7b-chat-codeCherryPop-qLoRA-GGML\" --tokenizer_name=\"TheBloke/llama2-7b-chat-codeCherryPop-qLoRA-GGML\" --model_type=\"ggml\" --save_history --file_name=\"llama-2-7b-chat-codeCherryPop.ggmlv3.q4_1.bin\"\r\n\r\necho \"Installation and setup completed successfully.\"","shortMessageHtmlLink":"Update automindx.install"}},{"before":"f8501fe0822fe855d0a996f7170df6ab9e0ced73","after":"749f2913f3e9a391e79732207443abeb68a02b81","ref":"refs/heads/main","pushedAt":"2024-04-23T20:18:31.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"THRUSTDeltaV","name":"DeltaV THRUST","path":"/THRUSTDeltaV","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/106721778?s=80&v=4"},"commit":{"message":"Update automindx.install\n\npython3 uiux.py --model_name=\"TheBloke/llama2-7b-chat-codeCherryPop-qLoRA-GGML\" --tokenizer_name=\"TheBloke/llama2-7b-chat-codeCherryPop-qLoRA-GGML\" --model_type=\"ggml\" --save_history --file_name=\"llama-2-7b-chat-codeCherryPop.ggmlv3.q4_1.bin\"","shortMessageHtmlLink":"Update automindx.install"}},{"before":"f5e946b6ca1c64ffd2ca68c490da6efc37bf068e","after":"f8501fe0822fe855d0a996f7170df6ab9e0ced73","ref":"refs/heads/main","pushedAt":"2024-04-21T03:27:34.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"THRUSTDeltaV","name":"DeltaV THRUST","path":"/THRUSTDeltaV","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/106721778?s=80&v=4"},"commit":{"message":"Update algm.md\n\naGLM for contextual memory storage","shortMessageHtmlLink":"Update algm.md"}},{"before":"79e34b730f7e786b08be29e3bc88f4016dbc4ef5","after":"f5e946b6ca1c64ffd2ca68c490da6efc37bf068e","ref":"refs/heads/main","pushedAt":"2023-09-04T19:44:05.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"Professor-Codephreak","name":"codephreak","path":"/Professor-Codephreak","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/140855987?s=80&v=4"},"commit":{"message":"Update automind.py\n\nAUTOMINDx","shortMessageHtmlLink":"Update automind.py"}},{"before":"fe9273f429bae0f8e184782b228f8969e8977f5e","after":"79e34b730f7e786b08be29e3bc88f4016dbc4ef5","ref":"refs/heads/main","pushedAt":"2023-09-01T21:51:36.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"Professor-Codephreak","name":"codephreak","path":"/Professor-Codephreak","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/140855987?s=80&v=4"},"commit":{"message":"Update 4096chunk.md\n\n# chunk4096.py<br />\r\nThis Python script creates a class chunk4096.py utilizing the Gradio library, PyTorch, and the Transformers library to perform text chunking and natural language generation. It defines a Chunker class responsible for breaking down input text into chunks and generating a response for each chunk. The chunk size is set to 4096 tokens.<br />\r\n\r\n# Dependencies<br />\r\nGradio<br />\r\nPyTorch<br />\r\nTransformers<br />\r\nConstants and Variables<br />\r\nBOS, EOS: Constants for Beginning Of Sentence and End Of Sentence tokens.\r\nB_INST, E_INST: Constants for beginning and ending an instance in the text.\r\nB_SYS, E_SYS: Constants for system-specific tokens.<br />","shortMessageHtmlLink":"Update 4096chunk.md"}},{"before":"0f8e8f9309b4d2dce1aee9df04e49a03d82011f7","after":"fe9273f429bae0f8e184782b228f8969e8977f5e","ref":"refs/heads/main","pushedAt":"2023-09-01T21:51:05.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"Professor-Codephreak","name":"codephreak","path":"/Professor-Codephreak","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/140855987?s=80&v=4"},"commit":{"message":"Update 4096chunk.md\n\n# Chunk4096.py<br />\r\nThis Python script creates a class chunk4096.py utilizing the Gradio library, PyTorch, and the Transformers library to perform text chunking and natural language generation. It defines a Chunker class responsible for breaking down input text into chunks and generating a response for each chunk. The chunk size is set to 4096 tokens.<br />","shortMessageHtmlLink":"Update 4096chunk.md"}},{"before":"d64c9e39023e29cce4d8849cde0cafb4a9660923","after":"0f8e8f9309b4d2dce1aee9df04e49a03d82011f7","ref":"refs/heads/main","pushedAt":"2023-09-01T21:49:49.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"Professor-Codephreak","name":"codephreak","path":"/Professor-Codephreak","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/140855987?s=80&v=4"},"commit":{"message":"Update 4096chunk.md","shortMessageHtmlLink":"Update 4096chunk.md"}},{"before":"bcae7f63732586882841eeab0d2ed462ddd26c74","after":"d64c9e39023e29cce4d8849cde0cafb4a9660923","ref":"refs/heads/main","pushedAt":"2023-09-01T21:48:26.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"Professor-Codephreak","name":"codephreak","path":"/Professor-Codephreak","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/140855987?s=80&v=4"},"commit":{"message":"Update 4096chunk.md\n\n<br />","shortMessageHtmlLink":"Update 4096chunk.md"}},{"before":"94e006397d0d48d1c50d9adb34a4d3d6a30c90fb","after":"bcae7f63732586882841eeab0d2ed462ddd26c74","ref":"refs/heads/main","pushedAt":"2023-09-01T21:46:29.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"Professor-Codephreak","name":"codephreak","path":"/Professor-Codephreak","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/140855987?s=80&v=4"},"commit":{"message":"Update 4096chunk.md\n\nChunk4096.py\r\nOverview\r\nThis Python script utilizes the Gradio library, PyTorch, and the Transformers library to perform text chunking and natural language generation. It defines a Chunker class responsible for breaking down input text into chunks and generating a response for each chunk. The chunk size is set to 4096 tokens.\r\n\r\nDependencies\r\nGradio\r\nPyTorch\r\nTransformers\r\nConstants and Variables\r\nBOS, EOS: Constants for Beginning Of Sentence and End Of Sentence tokens.\r\nB_INST, E_INST: Constants for beginning and ending an instance in the text.\r\nB_SYS, E_SYS: Constants for system-specific tokens.\r\nClasses\r\nChunker\r\nClass Attributes:\r\nCHUNK_SIZE: Defines the chunk size for the class. Set to 4096 tokens.\r\nInstance Attributes:\r\ntokenizer: Tokenizer object for text tokenization.\r\nmodel: Language model object for text generation.\r\nMethods:\r\nchunk_and_generate(chunk_text):\r\nInput: A string chunk_text.\r\nOutput: A generated string based on chunk_text.\r\nDescription: Tokenizes the input chunk and generates a text using the model.\r\nFunctions\r\nchunk_and_generate_response(chunker, text)\r\nInput:\r\nchunker: An instance of the Chunker class.\r\ntext: A string containing the text to be chunked and processed.\r\nOutput: A string containing the generated text.\r\nDescription: Splits the input text into chunks and utilizes chunker to generate text for each chunk. The generated text is then concatenated and returned.\r\nSecurity Measures\r\nInput Validation: Ensure that the chunked text is sanitized and valid.\r\nSensitive Data Protection: No sensitive data is handled in this script.\r\nLeast Privilege Principle: The script only uses the resources it needs.\r\nError Handling and Logging: Error handling can be implemented where the model or tokenizer is used to generate the text.","shortMessageHtmlLink":"Update 4096chunk.md"}},{"before":"cf62b23eafb39ff25fa941c182d98ef529094db2","after":"94e006397d0d48d1c50d9adb34a4d3d6a30c90fb","ref":"refs/heads/main","pushedAt":"2023-09-01T21:44:26.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"Professor-Codephreak","name":"codephreak","path":"/Professor-Codephreak","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/140855987?s=80&v=4"},"commit":{"message":"Update chunk4096.md","shortMessageHtmlLink":"Update chunk4096.md"}},{"before":"157a7aefd9345e6616eac09c2985b27ce6303c9e","after":"cf62b23eafb39ff25fa941c182d98ef529094db2","ref":"refs/heads/main","pushedAt":"2023-09-01T18:38:35.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"Professor-Codephreak","name":"codephreak","path":"/Professor-Codephreak","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/140855987?s=80&v=4"},"commit":{"message":"Add files via upload\n\nsolved chunk4096.py large input error using chunk4096.py as a chunking class","shortMessageHtmlLink":"Add files via upload"}},{"before":null,"after":"157a7aefd9345e6616eac09c2985b27ce6303c9e","ref":"refs/heads/main","pushedAt":"2023-09-01T18:36:48.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"Professor-Codephreak","name":"codephreak","path":"/Professor-Codephreak","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/140855987?s=80&v=4"},"commit":{"message":"Initial commit","shortMessageHtmlLink":"Initial commit"}}],"hasNextPage":false,"hasPreviousPage":false,"activityType":"all","actor":null,"timePeriod":"all","sort":"DESC","perPage":30,"cursor":"Y3Vyc29yOnYyOpK7MjAyNC0wNC0yNFQxODo0NjoyMi4wMDAwMDBazwAAAAQ5dJv4","startCursor":"Y3Vyc29yOnYyOpK7MjAyNC0wNC0yNFQxODo0NjoyMi4wMDAwMDBazwAAAAQ5dJv4","endCursor":"Y3Vyc29yOnYyOpK7MjAyMy0wOS0wMVQxODozNjo0OC4wMDAwMDBazwAAAAN3rsWz"}},"title":"Activity · pythaiml/automindx"}