From 608b7a725c6e1207578d794dfe75b6ededaca52b Mon Sep 17 00:00:00 2001 From: thePR0M3TH3AN <53631862+PR0M3TH3AN@users.noreply.github.com> Date: Sun, 20 Oct 2024 17:03:58 -0400 Subject: [PATCH] update --- .gitignore | 22 +++ README.md | 219 +++++++++++++++++++++ context_tool/config.yaml | 97 +++++++++ context_tool/generate_repo-context.py | 270 ++++++++++++++++++++++++++ context_tool/important_info.txt | 0 context_tool/overview.txt | 0 context_tool/prompt_templates.txt | 30 +++ context_tool/repo-context.txt | 0 context_tool/requirements.txt | 4 + context_tool/to-do_list.txt | 0 10 files changed, 642 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 context_tool/config.yaml create mode 100755 context_tool/generate_repo-context.py create mode 100644 context_tool/important_info.txt create mode 100644 context_tool/overview.txt create mode 100644 context_tool/prompt_templates.txt create mode 100644 context_tool/repo-context.txt create mode 100644 context_tool/requirements.txt create mode 100644 context_tool/to-do_list.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..556056e --- /dev/null +++ b/.gitignore @@ -0,0 +1,22 @@ +# Ignore virtual environment directory +venv/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# Encrypted index file should be backed up, hence not ignored +!.deterministic_password_generator/password_indices.csv + +# Ignore system files +.DS_Store +Thumbs.db + +# Ignore logs and temporary files +*.log +*.tmp + +# Python env +.env +*.env \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..4e357e5 --- /dev/null +++ b/README.md @@ -0,0 +1,219 @@ +# Repository Context Generator + +The **Repository Context Generator** is a tool designed to create a comprehensive context file (`repo-context.txt`) for AI coding assistants like ChatGPT. This context file aggregates essential information from your repository, including an overview, key details, a directory tree with exclusions, contents of important files with syntax highlighting, and a to-do list. This facilitates more informed and efficient interactions with AI assistants regarding your codebase. + +## Features + +- **Context File Generation**: Creates a `repo-context.txt` file tailored for AI assistants. +- **Configurable Directory Tree**: Generates a directory tree with the ability to exclude specified directories. +- **Highlighted File Contents**: Incorporates important files with syntax highlighting based on their file types. +- **Static Content Integration**: Adds static sections from files such as `overview.txt`, `important_info.txt`, and `to-do_list.txt`. +- **Extensible Configuration**: Easily customize exclusions, important files, and additional sections via `config.yaml`. + +## Table of Contents + +- [Features](#features) +- [Prerequisites](#prerequisites) +- [Setup Guide](#setup-guide) + - [Step 1: Clone the Repository](#step-1-clone-the-repository) + - [Step 2: Set Up Python Virtual Environment (Optional but Recommended)](#step-2-set-up-python-virtual-environment-optional-but-recommended) + - [Step 3: Install Required Packages](#step-3-install-required-packages) + - [Step 4: Configure the Script](#step-4-configure-the-script) + - [Step 5: Running the Script](#step-5-running-the-script) +- [Output](#output) +- [Customization](#customization) + - [Modifying `config.yaml`](#modifying-configyaml) +- [Additional Notes](#additional-notes) +- [Contributing](#contributing) +- [License](#license) + +## Prerequisites + +- **Python 3.7 or higher**: Ensure Python is installed on your system. You can download it from [python.org](https://www.python.org/downloads/). +- **Git**: To clone the repository. Download from [git-scm.com](https://git-scm.com/downloads). + +## Setup Guide + +### Step 1: Clone the Repository + +Clone this repository to your local machine using Git: + +```bash +git clone +cd +``` + +*Replace `` with the actual URL of your repository and `` with the cloned directory name.* + +### Step 2: Set Up Python Virtual Environment (Optional but Recommended) + +Using a virtual environment isolates your project's dependencies, preventing conflicts with other projects. + +1. **Create a Virtual Environment**: + + ```bash + python3 -m venv venv + ``` + +2. **Activate the Virtual Environment**: + + - **macOS and Linux**: + ```bash + source venv/bin/activate + ``` + + - **Windows (Command Prompt)**: + ```bash + venv\Scripts\activate.bat + ``` + + - **Windows (PowerShell)**: + ```bash + venv\Scripts\Activate.ps1 + ``` + +### Step 3: Install Required Packages + +Install the necessary Python packages using `pip`: + +```bash +pip install -r requirements.txt +``` + +### Step 4: Configure the Script + +Customize the `config.yaml` file to control which directories and files are included or excluded in the generated context. + +1. **Locate `config.yaml`**: It's in the root directory of the cloned repository. + +2. **Edit `config.yaml`**: + - **Exclude Directories**: Modify the `exclude_dirs` section to exclude any directories you don't want in the context. + - **Important Files**: List the key files under the `important_files` section that should be included with their content. + +*Refer to the [Customization](#customization) section for detailed instructions.* + +### Step 5: Running the Script + +Execute the script to generate the `repo-context.txt` file. + +- **Unix-like Systems (macOS, Linux)**: + + ```bash + chmod +x generate_repo-context.py # Make the script executable (optional) + ./generate_repo-context.py + ``` + + Or simply: + + ```bash + python3 generate_repo-context.py + ``` + +- **Windows**: + + ```bash + python generate_repo-context.py + ``` + +## Output + +After running the script, a `repo-context.txt` file will be generated in the current directory. This file includes the following sections: + +- **Overview**: Content from `overview.txt` +- **Important Information**: Content from `important_info.txt` +- **Directory Tree**: Structure of your project with specified exclusions +- **Important Files**: Contents of key files with syntax highlighting +- **To-Do List**: Content from `to-do_list.txt` + +## Customization + +### Modifying `config.yaml` + +The `config.yaml` file allows you to tailor the context generation process to your project's needs. + +#### 1. **Exclude Directories** + +Specify directories that should be omitted from the directory tree and file inclusions. + +```yaml +exclude_dirs: + - node_modules # Node.js dependencies + - venv # Python virtual environment + - __pycache__ # Python bytecode cache + - build # Build output directories + - dist # Distribution packages + - .git # Git repository metadata + - .github # GitHub workflows and configurations + - .vscode # Visual Studio Code settings + - logs # Log files + - tmp # Temporary files and directories +``` + +*Add or remove directories as needed.* + +#### 2. **Important Files** + +List the crucial files whose content should be included in the context file. Paths should be relative to the main source directory (default is `src/`). + +```yaml +important_files: + - main.py # Entry point of the application + - app.py # Application configuration + - config/settings.py # Configuration settings + - utils/helpers.py # Utility helper functions + - models/user.py # User model definitions + - controllers/auth_controller.py# Authentication controller + - services/email_service.py # Email service integration + - routes/api_routes.py # API route definitions + - database/db_connection.py # Database connection setup + - tests/test_main.py # Main application tests +``` + +*Update the list based on your project's structure.* + +#### 3. **Additional Configuration (Optional)** + +Uncomment and customize additional sections for more advanced configurations. + +```yaml +# List of file types to include based on extensions. +file_type_inclusions: + - .js + - .ts + - .java + - .rb + - .go + +# List of file types to exclude based on extensions. +file_type_exclusions: + - .log + - .tmp + - .png + - .jpg + - .gif + +# Custom sections to include additional information. +custom_sections: + - file: changelog.txt + section_title: "Changelog" + - file: LICENSE.txt + section_title: "License" +``` + +*Customize these sections as per your project requirements.* + +## Additional Notes + +- **Static Files**: Ensure that `overview.txt`, `important_info.txt`, and `to-do_list.txt` are present in the same directory as `generate_repo-context.py`. +- **Syntax Highlighting**: The script supports syntax highlighting for common file types like `.py`, `.js`, `.json`, etc. To add more file types, update the `LANGUAGE_MAP` in the script. +- **Source Directory**: By default, the script assumes your main source code is in the `src/` directory. If your project uses a different structure, update the `start_path` in the script or make it configurable. + +## Contributing + +Contributions are welcome! Please open an issue or submit a pull request for any enhancements or bug fixes. + +## License + +This project is licensed under the [MIT License](LICENSE). + +--- diff --git a/context_tool/config.yaml b/context_tool/config.yaml new file mode 100644 index 0000000..c57eeb8 --- /dev/null +++ b/context_tool/config.yaml @@ -0,0 +1,97 @@ +# ========================================= +# AI Code Context Generator Configuration +# ========================================= + +# Primary source directory containing the main codebase. +# Update this if your main code is not in 'src/'. +source_directory: src + +# List of directories to exclude from the directory tree and file inclusions. +# These directories and their contents will be omitted from the generated context. +# Customize this list based on your project's structure. +exclude_dirs: + - node_modules # Node.js dependencies + - venv # Python virtual environment + - __pycache__ # Python bytecode cache + - build # Build output directories + - dist # Distribution packages + - .git # Git repository metadata + - .github # GitHub workflows and configurations + - .vscode # Visual Studio Code settings + - logs # Log files + - tmp # Temporary files and directories + +# List of important files to include in the context. +# Paths should be relative to the 'source_directory' specified above. +# These files are crucial for understanding the project's functionality. +# Modify this list to include files relevant to your project. +important_files: + - main.py # Entry point of the application + - app.py # Application configuration + - config/settings.py # Configuration settings + - utils/helpers.py # Utility helper functions + - models/user.py # User model definitions + - controllers/auth_controller.py # Authentication controller + - services/email_service.py # Email service integration + - routes/api_routes.py # API route definitions + - database/db_connection.py # Database connection setup + - tests/test_main.py # Main application tests + +# ========================================= +# Instructions for Customization +# ========================================= + +# 1. **source_directory**: +# - Set this to the primary directory containing your source code. +# - Example: +# - For a project with main code in 'app/', set `source_directory: app` + +# 2. **exclude_dirs**: +# - Review the list and remove any directories that are essential for your project context. +# - Add any additional directories that you want to exclude by appending them to the list. +# - Example: +# - If your project uses a `docs/` directory for documentation, you might choose to exclude it: +# - docs + +# 3. **important_files**: +# - Identify the key files in your project that define its core functionality. +# - Ensure the paths are relative to your `source_directory`. +# - Add or remove files as necessary to reflect your project's structure. +# - Example: +# - For a JavaScript project, you might include: +# - index.js +# - src/app.js +# - src/routes/index.js +# - src/controllers/userController.js + +# ========================================= +# Additional Configuration (Optional) +# ========================================= + +# Uncomment and customize the sections below if your project requires more advanced configurations. + +# # List of file types to include based on extensions. +# # Useful if you want to include all files of certain types without specifying each one. +# file_type_inclusions: +# - .js +# - .ts +# - .java +# - .rb +# - .go + +# # List of file types to exclude based on extensions. +# # This can help in omitting large binary files or irrelevant file types. +# file_type_exclusions: +# - .log +# - .tmp +# - .png +# - .jpg +# - .gif + +# # Custom sections to include additional information. +# # You can define more sections by adding corresponding static files. +custom_sections: + - file: changelog.txt + section_title: "Changelog" + - file: LICENSE.txt + section_title: "License" diff --git a/context_tool/generate_repo-context.py b/context_tool/generate_repo-context.py new file mode 100755 index 0000000..31b030d --- /dev/null +++ b/context_tool/generate_repo-context.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 + +""" +Script Name: generate_repo-context.py +Description: Generates a context file (`repo-context.txt`) for AI coding assistants. + Includes an overview, important information, a directory tree with exclusions, + content of important files with syntax highlighting, and a to-do list. + +Usage: +1. Ensure you have Python 3.7 or higher installed. +2. (Optional) Set up a Python virtual environment: + python3 -m venv venv + source venv/bin/activate # On Unix or MacOS + venv\Scripts\activate.bat # On Windows (Command Prompt) + venv\Scripts\Activate.ps1 # On Windows (PowerShell) +3. Install the required Python packages: + pip install -r requirements.txt +4. Configure `config.yaml` as needed. +5. Place `overview.txt`, `important_info.txt`, and `to-do_list.txt` in the script directory. +6. Run the script: + ./generate_repo-context.py # Unix-like systems + python generate_repo-context.py # Windows + +The script will create `repo-context.txt` with the specified structure. +""" + +import os +import sys +import yaml +from pathlib import Path +import mimetypes +import logging +from typing import List, Dict + +# Configuration Constants +CONFIG_FILE = "config.yaml" +OUTPUT_FILE = "repo-context.txt" + +# Static Text Files and Their Corresponding Section Titles +STATIC_FILES = [ + {"file": "overview.txt", "section_title": "Overview"}, + {"file": "important_info.txt", "section_title": "Important Information"}, + {"file": "to-do_list.txt", "section_title": "To-Do List"} +] + +# Mapping of File Extensions to Programming Languages for Syntax Highlighting +LANGUAGE_MAP = { + '.py': 'python', + '.json': 'json', + '.env': 'bash', + '.js': 'javascript', + '.html': 'html', + '.css': 'css', + '.csv': 'csv', + '.md': 'markdown', + '.txt': '', # Plain text + # Add more mappings as needed +} + +# Extensions of Binary Files to Skip +BINARY_EXTENSIONS = ['.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico', '.db', '.exe', '.bin'] + +def setup_logging(): + """Configures the logging format and level.""" + logging.basicConfig( + level=logging.INFO, + format='[%(levelname)s] %(message)s' + ) + +def load_config(config_path: Path) -> Dict: + """ + Loads configuration from a YAML file. + + Args: + config_path (Path): Path to the YAML configuration file. + + Returns: + dict: Configuration dictionary containing 'exclude_dirs' and 'important_files'. + """ + if not config_path.exists(): + logging.error(f"Configuration file {config_path} not found.") + sys.exit(1) + try: + with open(config_path, 'r') as f: + config = yaml.safe_load(f) + logging.info(f"Loaded configuration from {config_path}.") + return config + except yaml.YAMLError as e: + logging.error(f"Error parsing configuration file: {e}") + sys.exit(1) + +def generate_directory_tree(start_path: Path, exclude_dirs: List[str]) -> List[str]: + """ + Generates a directory tree as a list of strings, excluding specified directories. + + Args: + start_path (Path): The root directory to start generating the tree from. + exclude_dirs (list): List of directory patterns to exclude. + + Returns: + list: List of strings representing the directory tree. + """ + tree_lines = [] + root = start_path.resolve() + for dirpath, dirnames, filenames in os.walk(start_path): + current_path = Path(dirpath) + rel_path = current_path.relative_to(root) + + # Skip excluded directories + if any(current_path.match(excl) or excl in rel_path.parts for excl in exclude_dirs): + dirnames[:] = [] # Don't traverse further into subdirectories + continue + + # Determine the indentation level + depth = len(rel_path.parts) + indent = " " * depth + connector = "├── " if depth > 0 else "." + tree_lines.append(f"{indent}{connector}{current_path.name}/" if depth > 0 else f"{connector}") + + # Add files in the current directory + for filename in sorted(filenames): + file_rel_path = rel_path / filename + if any(file_rel_path.match(excl) or excl in file_rel_path.parts for excl in exclude_dirs): + continue + file_indent = " " * (depth + 1) + tree_lines.append(f"{file_indent}├── {filename}") + + logging.info("Directory tree generated.") + return tree_lines + +def write_directory_tree(tree_lines: List[str], output_file: Path): + """ + Writes the directory tree to the output file within markdown code blocks. + + Args: + tree_lines (list): List of strings representing the directory tree. + output_file (Path): Path to the output file where the tree will be written. + """ + output_file.write_text("## Directory Tree with Exclusions\n\n```\n") + output_file.write("\n".join(tree_lines)) + output_file.write("\n```\n\n") + logging.info("Directory tree written to the context file.") + +def write_file_content(file_path: Path, output_file: Path): + """ + Writes the content of a file to the output file within markdown code blocks with syntax highlighting. + + Args: + file_path (Path): Path to the file whose content is to be written. + output_file (Path): Path to the output file where the content will be written. + """ + ext = file_path.suffix + language = LANGUAGE_MAP.get(ext, '') + relative_display_path = file_path.relative_to(file_path.parents[1]) + + output_file.write(f"## {relative_display_path}\n") + if language: + output_file.write(f"```{language}\n") + else: + output_file.write("```\n") + try: + if ext in BINARY_EXTENSIONS: + output_file.write(f"*Binary file ({ext}) cannot be displayed.*\n") + else: + content = file_path.read_text(encoding='utf-8', errors='ignore') + output_file.write(content) + except Exception as e: + output_file.write(f"*Error reading file: {e}*\n") + output_file.write("\n```\n\n") + logging.info(f"Included content from {file_path}.") + +def write_static_file(file_path: Path, output_file: Path, section_title: str): + """ + Writes the content of a static text file to the output file with a section header. + + Args: + file_path (Path): Path to the static text file. + output_file (Path): Path to the output file where the content will be written. + section_title (str): Title of the section to be added before the content. + """ + if not file_path.exists(): + logging.warning(f"Static file {file_path} not found, skipping...") + return + output_file.write(f"## {section_title}\n\n") + try: + content = file_path.read_text(encoding='utf-8', errors='ignore') + output_file.write(content + "\n\n") + logging.info(f"Included static section: {section_title}.") + except Exception as e: + output_file.write(f"*Error reading {file_path.name}: {e}*\n\n") + logging.error(f"Error reading {file_path}: {e}") + +def write_custom_sections(custom_sections: List[Dict], script_dir: Path, output_file: Path): + """ + Writes custom sections to the output file based on configuration. + + Args: + custom_sections (list): List of dictionaries with 'file' and 'section_title'. + script_dir (Path): Directory where the script is located. + output_file (Path): Path to the output file. + """ + for section in custom_sections: + file_name = section.get('file') + section_title = section.get('section_title', 'Custom Section') + file_path = script_dir / file_name + write_static_file(file_path, output_file, section_title) + +def main(): + """Main function that orchestrates the generation of the repository context file.""" + setup_logging() + + # Determine the script's directory + script_dir = Path(__file__).parent.resolve() + + # Load configuration + config_path = script_dir / CONFIG_FILE + config = load_config(config_path) + exclude_dirs = config.get("exclude_dirs", []) + important_files = config.get("important_files", []) + custom_sections = config.get("custom_sections", []) + + # Define the starting path (default to 'src' directory or as specified) + source_dir = config.get("source_directory", "src") + start_path = script_dir.parent / source_dir + if not start_path.exists(): + logging.error(f"Source directory {start_path} does not exist.") + sys.exit(1) + + output_file = script_dir / OUTPUT_FILE + output_file.unlink(missing_ok=True) # Remove if exists + + # Write a header to the output file + with output_file.open('w', encoding='utf-8') as f: + f.write(f"# Repository Context\n\n") + f.write(f"Generated on: {Path.now().strftime('%Y-%m-%d')}\n\n") + + # Write static sections + for static in STATIC_FILES: + static_path = script_dir / static["file"] + write_static_file(static_path, output_file, static["section_title"]) + + # Generate and write the directory tree + tree_lines = generate_directory_tree(start_path, exclude_dirs) + write_directory_tree(tree_lines, output_file) + + # Write important files + output_file.write("## Important Files\n\n") + for relative_file in important_files: + file_path = start_path / relative_file + if file_path.exists(): + write_file_content(file_path, output_file) + else: + output_file.write(f"*File `{relative_file}` not found, skipping...*\n\n") + logging.warning(f"Important file {relative_file} not found, skipping...") + + # Write custom sections if any + if custom_sections: + write_custom_sections(custom_sections, script_dir, output_file) + + # Write to-do list + for static in STATIC_FILES: + if static["section_title"] == "To-Do List": + continue # Already written + todo_path = script_dir / "to-do_list.txt" + write_static_file(todo_path, output_file, "To-Do List") + + logging.info(f"Context file created: {output_file}") + +if __name__ == "__main__": + main() diff --git a/context_tool/important_info.txt b/context_tool/important_info.txt new file mode 100644 index 0000000..e69de29 diff --git a/context_tool/overview.txt b/context_tool/overview.txt new file mode 100644 index 0000000..e69de29 diff --git a/context_tool/prompt_templates.txt b/context_tool/prompt_templates.txt new file mode 100644 index 0000000..9b3a192 --- /dev/null +++ b/context_tool/prompt_templates.txt @@ -0,0 +1,30 @@ +######################################################### +# o1-Preview to o1-Mini hand-off +######################################################### +######################################################### + +Can you please update my existing code with the recommendations below it? + +Here is the original code: + + + +``` + + +``` + +And here is the update we need to incorporate: + +--- + + + +``` + + +``` + +######################################################### +######################################################### + diff --git a/context_tool/repo-context.txt b/context_tool/repo-context.txt new file mode 100644 index 0000000..e69de29 diff --git a/context_tool/requirements.txt b/context_tool/requirements.txt new file mode 100644 index 0000000..39e3ebe --- /dev/null +++ b/context_tool/requirements.txt @@ -0,0 +1,4 @@ +# requirements.txt + +# PyYAML is required for parsing YAML configuration files +PyYAML>=6.0 diff --git a/context_tool/to-do_list.txt b/context_tool/to-do_list.txt new file mode 100644 index 0000000..e69de29