Skip to content

Commit 22a0ccd

Browse files
pj-simpsonpj-simpson
authored
Merge pull request #1688 from codatio/refactoring-docker-bits
Refactoring the code checker functionality
2 parents 6043c0e + 58eb875 commit 22a0ccd

File tree

20 files changed

+1167
-254
lines changed

20 files changed

+1167
-254
lines changed

‎code_utils/README.md‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@ uv sync --extra dev
2323
# Run the code extractor
2424
uv run code-util extract
2525

26-
# Run compiler checks on extracted code
27-
uv run code-util check
26+
# Run compiler checks on extracted code, (can only be done on lang at a time)
27+
uv run code-util check -l programming-language
2828

2929
# Get help in the cli
3030
uv run code-util --help
@@ -34,14 +34,21 @@ uv run code-util --help
3434
## Structure
3535

3636
- `main.py` - Entrypoint for the CLI.
37-
- `code_finder.py` - CodeFinder class
38-
- `code_checker.py` - CodeChecker class.
39-
- `docker/` - docker files are different scripts and config the container uses.
37+
- `code_finder/` - module for code relating to the CodeFinder class a.k.a the functionality to scan through markdown documents.
38+
- `code_checker/` - module for code relating to the CodeChecker class a.k.a the functionality to build containers and run commands in them.
39+
- `code_checker/docker/` - docker files, scripts and config that the containers use.
4040
- `temp/` - Generated code snippets (gitignored).
4141
- `files_with_code.txt` - List of files containing code (gitignored).
4242

4343
## Dependancies
4444

45-
- [Click](https://click.palletsprojects.com/en/stable/) - Framework for building CLIs
45+
- [Click](https://click.palletsprojects.com/en/stable/) - Framework for building CLIs.
4646
- [Docker SDK For Python](https://docker-py.readthedocs.io/en/stable/) - Library for interacting with the Docker API.
47+
- [Pytest](https://docs.pytest.org/en/stable/index.html) - Python Unit testing Framework.
48+
- [Pyfakefs](https://pytest-pyfakefs.readthedocs.io/en/latest/) - Python utility for faking a filesystem during testing.
49+
50+
## Notes
51+
52+
- Hard coded to only deal with python, javascript(typescript) and C#
53+
- Javascript container uses a private npm package. Please set PAT_TOKEN and CODAT_EMAIL env vars in order to build.
4754

‎code_utils/code_checker/__init__.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
"""Code checker package for validating code snippets from Codat documentation."""
2+
3+
from .code_checker import CodeChecker
4+
from .docker_operator import DockerOperator
5+
from .codat_code_checker_config_models import CodeCheckerConfig, LanguageDockerConfig, DEFAULT_CONFIG
6+
7+
__all__ = ['CodeChecker', 'DockerOperator', 'CodeCheckerConfig', 'LanguageDockerConfig', 'DEFAULT_CONFIG']

‎code_utils/code_checker/codat_code_checker_config_models.py‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
from dataclasses import dataclass
2+
from typing import Dict
3+
from pathlib import Path
4+
5+
6+
@dataclass
7+
class LanguageDockerConfig:
8+
"""Configuration for a specific programming language's Docker setup."""
9+
docker_directory: str
10+
validation_command: str
11+
working_directory: str = ""
12+
13+
def get_docker_path(self, base_dir: Path) -> Path:
14+
"""Get the full path to the Docker directory for this language."""
15+
return base_dir / self.docker_directory
16+
17+
18+
@dataclass
19+
class CodeCheckerConfig:
20+
"""Complete configuration for the CodeChecker Docker utility."""
21+
python: LanguageDockerConfig
22+
javascript: LanguageDockerConfig
23+
csharp: LanguageDockerConfig
24+
container_name: str = "code-snippets:latest"
25+
26+
def get_language_config(self, language: str) -> LanguageDockerConfig:
27+
"""Get Docker configuration for a specific language."""
28+
language_map = {
29+
'python': self.python,
30+
'javascript': self.javascript,
31+
'csharp': self.csharp
32+
}
33+
return language_map.get(language.lower())
34+
35+
def get_all_languages(self) -> list[str]:
36+
"""Get list of all supported language names."""
37+
return ['python', 'javascript', 'csharp']
38+
39+
40+
# Default configuration instance based on current CodeChecker implementation
41+
DEFAULT_CONFIG = CodeCheckerConfig(
42+
python=LanguageDockerConfig(
43+
docker_directory="docker/python",
44+
validation_command="bash -c 'cd snippets && pyright .'",
45+
working_directory="python/snippets"
46+
),
47+
javascript=LanguageDockerConfig(
48+
docker_directory="docker/javascript",
49+
validation_command="tsc --noEmit",
50+
working_directory="javascript"
51+
),
52+
csharp=LanguageDockerConfig(
53+
docker_directory="docker/csharp",
54+
validation_command="bash -c 'cd /workspace/code-snippets/csharp && ./validate-csharp-snippets.sh'",
55+
working_directory="/workspace/code-snippets/csharp"
56+
),
57+
container_name="code-snippets"
58+
)

‎code_utils/code_checker/code_checker.py‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
"""
2+
Code Checker for validating code snippets for a specified programming language.
3+
This module builds and runs a Docker container to validate complete code snippets.
4+
"""
5+
6+
import sys
7+
from pathlib import Path
8+
from typing import Dict, Optional
9+
10+
from .codat_code_checker_config_models import CodeCheckerConfig, DEFAULT_CONFIG
11+
from .docker_operator import DockerOperator
12+
13+
14+
class CodeChecker:
15+
"""
16+
A class for validating code snippets by building and running validation commands
17+
in a Docker container for a specified programming language environment.
18+
"""
19+
20+
def __init__(self, target_language: str, config: Optional[CodeCheckerConfig] = None, base_dir: Optional[Path] = None, docker_operator: Optional[DockerOperator] = None):
21+
"""
22+
Initialize the CodeChecker with dependency injection.
23+
24+
Args:
25+
target_language: Specific language to check (required).
26+
config: Configuration object for Docker settings. Defaults to DEFAULT_CONFIG.
27+
base_dir: Base directory path. Defaults to the directory containing this file.
28+
docker_operator: DockerOperator instance. If None, creates DockerOperator(config, base_dir).
29+
"""
30+
self.config = config or DEFAULT_CONFIG
31+
self.base_dir = base_dir or Path(__file__).parent
32+
self.docker_wrapper = docker_operator or DockerOperator(self.config, self.base_dir)
33+
self.target_language = target_language.lower()
34+
35+
def _build_docker_images(self) -> Dict[str, Dict[str, any]]:
36+
"""
37+
Build Docker image for the target language.
38+
39+
Returns:
40+
Dictionary with build results for the target language
41+
"""
42+
build_results = {}
43+
44+
print(f"🔨 Building Docker images for: {self.target_language}...")
45+
print("=" * 60)
46+
47+
success, message = self.docker_wrapper.build_language_image(self.target_language)
48+
build_results[self.target_language] = {
49+
'success': success,
50+
'message': message
51+
}
52+
53+
if not success:
54+
print(f"❌ Failed to build {self.target_language} image: {message}")
55+
56+
print("=" * 60)
57+
58+
return build_results
59+
60+
def _validate_language_snippets(self, language: str) -> Dict[str, any]:
61+
"""
62+
Validate code snippets for a specific language using the DockerOperator.
63+
64+
Args:
65+
language: The programming language to validate
66+
67+
Returns:
68+
Dictionary with validation results
69+
"""
70+
success, output = self.docker_wrapper.validate_language_snippets(language)
71+
return {
72+
'success': success,
73+
'output': output
74+
}
75+
76+
def check_complete_snippets(self) -> Dict[str, Dict[str, any]]:
77+
"""
78+
Build Docker image and validate complete code snippets for the target language.
79+
80+
Returns:
81+
Dictionary with validation results for the target language:
82+
{
83+
'build': {language: {'success': bool, 'message': str}},
84+
'validation': {language: {'success': bool, 'output': str}}
85+
}
86+
"""
87+
print(f"🚀 Starting code snippet validation for: {self.target_language}...")
88+
print("=" * 60)
89+
90+
# Step 1: Build Docker images for target languages
91+
build_results = self._build_docker_images()
92+
93+
# Check if all builds succeeded
94+
all_builds_successful = all(result['success'] for result in build_results.values())
95+
96+
if not all_builds_successful:
97+
failed_builds = [lang for lang, result in build_results.items() if not result['success']]
98+
print(f"❌ Docker build failed for: {', '.join(failed_builds)}")
99+
return {
100+
'build': build_results,
101+
'validation': {}
102+
}
103+
104+
print("=" * 60)
105+
106+
# Step 2: Validate snippets for target languages
107+
validation_results = {}
108+
109+
validation_results[self.target_language] = self._validate_language_snippets(self.target_language)
110+
111+
# Summary
112+
print("=" * 60)
113+
print("📊 Validation Summary:")
114+
115+
validation_result = validation_results[self.target_language]
116+
117+
if validation_result['success']:
118+
print(f"🎉 {self.target_language.title()} validation passed!")
119+
else:
120+
print(f"❌ {self.target_language.title()} validation failed!")
121+
122+
return {
123+
'build': build_results,
124+
'validation': validation_results
125+
}
126+
127+
128+

‎code_utils/code_checker/docker/.dockerignore‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
# Exclude unnecessary files from Docker build context
2+
**/__pycache__/
3+
**/*.pyc
4+
**/*.pyo
5+
**/*.pyd
6+
**/.Python
7+
**/env/
8+
**/venv/
9+
**/.venv/
10+
**/pip-log.txt
11+
**/pip-delete-this-directory.txt
12+
13+
# Node modules and TypeScript compilation outputs
14+
**/node_modules/
15+
**/npm-debug.log*
16+
**/yarn-debug.log*
17+
**/yarn-error.log*
18+
**/.npm
19+
**/.yarn-integrity
20+
**/dist/
21+
**/build/
22+
23+
# .NET build outputs
24+
**/bin/
25+
**/obj/
26+
**/*.user
27+
**/*.suo
28+
**/*.cache
29+
**/packages/
30+
31+
# Version control
32+
**/.git/
33+
**/.gitignore
34+
**/.gitattributes
35+
36+
# IDE and editor files
37+
**/.vscode/
38+
**/.idea/
39+
**/*.swp
40+
**/*.swo
41+
*~
42+
43+
# OS generated files
44+
**/.DS_Store
45+
**/.DS_Store?
46+
**/._*
47+
**/.Spotlight-V100
48+
**/.Trashes
49+
**/ehthumbs.db
50+
**/Thumbs.db
51+
52+
# Documentation and temp files
53+
**/temp/incomplete/
54+
**/*.md
55+
**/*.txt
56+
**/output.txt
57+
**/link-results.json
58+
59+
# Only include the complete code snippets and dependency files
60+
!temp/*/complete/
61+
!docker/requirements.txt
62+
!docker/package.json
63+
!docker/tsconfig.json
64+
!docker/CodatSnippets.csproj

‎code_utils/code_checker/docker/csharp/CodatSnippets.csproj‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<TargetFramework>net8.0</TargetFramework>
5+
<OutputType>Exe</OutputType>
6+
<ImplicitUsings>enable</ImplicitUsings>
7+
<Nullable>enable</Nullable>
8+
</PropertyGroup>
9+
10+
<ItemGroup>
11+
<PackageReference Include="Codat.Platform" Version="*" />
12+
<PackageReference Include="Codat.BankFeeds" Version="*" />
13+
<PackageReference Include="Codat.Lending" Version="*" />
14+
<PackageReference Include="Codat.Sync.Commerce" Version="*" />
15+
<PackageReference Include="Codat.Sync.Expenses" Version="*" />
16+
<PackageReference Include="Codat.Sync.Payables" Version="*" />
17+
<PackageReference Include="Codat.Sync.Payroll" Version="*" />
18+
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
19+
</ItemGroup>
20+
21+
</Project>

‎code_utils/code_checker/docker/csharp/Dockerfile‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
# C#/.NET-specific Dockerfile for Codat code snippets validation
2+
FROM ubuntu:22.04
3+
4+
# Set environment variables
5+
ENV DEBIAN_FRONTEND=noninteractive
6+
ENV DOTNET_CLI_TELEMETRY_OPTOUT=1
7+
ENV DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
8+
9+
# Update package list and install common dependencies
10+
RUN apt-get update && apt-get install -y \
11+
curl \
12+
wget \
13+
apt-transport-https \
14+
software-properties-common \
15+
gnupg \
16+
lsb-release \
17+
ca-certificates \
18+
&& rm -rf /var/lib/apt/lists/*
19+
20+
# Install .NET 8.0 SDK
21+
RUN wget https://packages.microsoft.com/config/ubuntu/22.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb \
22+
&& dpkg -i packages-microsoft-prod.deb \
23+
&& rm packages-microsoft-prod.deb \
24+
&& apt-get update \
25+
&& apt-get install -y dotnet-sdk-8.0 \
26+
&& rm -rf /var/lib/apt/lists/*
27+
28+
# Create workspace directory
29+
WORKDIR /workspace/code-snippets/csharp
30+
31+
# Copy C# project file and restore packages
32+
COPY code_checker/docker/csharp/CodatSnippets.csproj ./CodatSnippets.csproj
33+
RUN dotnet restore
34+
35+
# Create directory for code snippets and copy them from temp directory
36+
RUN mkdir -p ./snippets/
37+
38+
# Copy C# code snippets from temp directory (similar to other languages)
39+
COPY temp/csharp/complete/ ./snippets/
40+
41+
# Copy C# validation script and fix line endings
42+
COPY code_checker/docker/csharp/validate-csharp-snippets.sh ./validate-csharp-snippets.sh
43+
RUN sed -i 's/\r$//' validate-csharp-snippets.sh && chmod +x validate-csharp-snippets.sh
44+
45+
# Verify .NET installation and show packages
46+
RUN echo "=== C#/.NET Environment Information ===" && \
47+
echo ".NET version:" && dotnet --version && \
48+
echo "" && \
49+
echo "=== C# Codat Packages ===" && \
50+
dotnet list package | grep -i codat || echo "Codat packages will be available after restore" && \
51+
echo "" && \
52+
echo "=== C# Snippets Count ===" && \
53+
find ./snippets -name "*.cs" 2>/dev/null | wc -l | xargs echo "C# snippets found:" || echo "No snippets directory found yet"
54+
55+
# Set working directory
56+
WORKDIR /workspace/code-snippets/csharp
57+
58+
# Default command
59+
CMD ["/bin/bash"]

0 commit comments

Comments
 (0)