feat: generate SDK for Remnawave API v2.7.4
This commit is contained in:
Binary file not shown.
Executable
+401
@@ -0,0 +1,401 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Create consolidated OpenAPI schema by extracting schemas section
|
||||
and applying consolidation mappings based on detected duplicate patterns.
|
||||
|
||||
This tool works with potentially malformed JSON files by extracting only
|
||||
the schemas section and rebuilding a clean OpenAPI spec.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def extract_schemas_section(filepath: str) -> dict:
|
||||
"""Extract only the schemas section from a potentially malformed OpenAPI file."""
|
||||
with open(filepath, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
|
||||
schemas_start = content.find('"schemas": {')
|
||||
if schemas_start < 0:
|
||||
raise ValueError('Could not find "schemas" section')
|
||||
|
||||
schemas_part = content[schemas_start + len('"schemas": '):]
|
||||
|
||||
# Count braces to find the end
|
||||
brace_count = 0
|
||||
in_string = False
|
||||
escape = False
|
||||
end_pos = 0
|
||||
|
||||
for i, char in enumerate(schemas_part):
|
||||
if escape:
|
||||
escape = False
|
||||
continue
|
||||
if char == '\\':
|
||||
escape = True
|
||||
continue
|
||||
if char == '"':
|
||||
in_string = not in_string
|
||||
continue
|
||||
if not in_string:
|
||||
if char == '{':
|
||||
brace_count += 1
|
||||
elif char == '}':
|
||||
brace_count -= 1
|
||||
if brace_count == 0:
|
||||
end_pos = i + 1
|
||||
break
|
||||
|
||||
schemas_json = schemas_part[:end_pos]
|
||||
wrapped = '{"schemas": ' + schemas_json + '}'
|
||||
data = json.loads(wrapped)
|
||||
return data['schemas']
|
||||
|
||||
|
||||
def create_consolidation_map() -> dict:
|
||||
"""
|
||||
Create consolidation map based on analyzed duplicate patterns.
|
||||
Maps: old_schema_name -> canonical_schema_name
|
||||
"""
|
||||
return {
|
||||
# Group 1: User Responses (9 duplicates)
|
||||
'DisableUserResponseDto': 'CreateUserResponseDto',
|
||||
'EnableUserResponseDto': 'CreateUserResponseDto',
|
||||
'GetUserByShortUuidResponseDto': 'CreateUserResponseDto',
|
||||
'GetUserByUsernameResponseDto': 'CreateUserResponseDto',
|
||||
'GetUserByUuidResponseDto': 'CreateUserResponseDto',
|
||||
'ResetUserTrafficResponseDto': 'CreateUserResponseDto',
|
||||
'RevokeUserSubscriptionResponseDto': 'CreateUserResponseDto',
|
||||
'UpdateUserResponseDto': 'CreateUserResponseDto',
|
||||
|
||||
# Group 2: Delete Operations (8 duplicates)
|
||||
'DeleteConfigProfileResponseDto': 'DeleteResponseDto',
|
||||
'DeleteExternalSquadResponseDto': 'DeleteResponseDto',
|
||||
'DeleteHostResponseDto': 'DeleteResponseDto',
|
||||
'DeleteInfraProviderByUuidResponseDto': 'DeleteResponseDto',
|
||||
'DeleteInternalSquadResponseDto': 'DeleteResponseDto',
|
||||
'DeleteNodeResponseDto': 'DeleteResponseDto',
|
||||
'DeleteSubscriptionTemplateResponseDto': 'DeleteResponseDto',
|
||||
'DeleteUserResponseDto': 'DeleteResponseDto',
|
||||
|
||||
# Group 3: Event Operations (8 duplicates)
|
||||
'AddUsersToExternalSquadResponseDto': 'EventResponseDto',
|
||||
'AddUsersToInternalSquadResponseDto': 'EventResponseDto',
|
||||
'BulkAllResetTrafficUsersResponseDto': 'EventResponseDto',
|
||||
'BulkAllUpdateUsersResponseDto': 'EventResponseDto',
|
||||
'RemoveUsersFromExternalSquadResponseDto': 'EventResponseDto',
|
||||
'RemoveUsersFromInternalSquadResponseDto': 'EventResponseDto',
|
||||
'RestartAllNodesResponseDto': 'EventResponseDto',
|
||||
'RestartNodeResponseDto': 'EventResponseDto',
|
||||
|
||||
# Group 4: Bulk Operations Response (6 duplicates)
|
||||
'BulkDeleteUsersByStatusResponseDto': 'BulkActionResponseDto',
|
||||
'BulkDeleteUsersResponseDto': 'BulkActionResponseDto',
|
||||
'BulkResetTrafficUsersResponseDto': 'BulkActionResponseDto',
|
||||
'BulkRevokeUsersSubscriptionResponseDto': 'BulkActionResponseDto',
|
||||
'BulkUpdateUsersResponseDto': 'BulkActionResponseDto',
|
||||
'BulkUpdateUsersSquadsResponseDto': 'BulkActionResponseDto',
|
||||
|
||||
# Group 5: Bulk Request (6 duplicates)
|
||||
'BulkDeleteHostsRequestDto': 'BulkUuidsRequestDto',
|
||||
'BulkDisableHostsRequestDto': 'BulkUuidsRequestDto',
|
||||
'BulkEnableHostsRequestDto': 'BulkUuidsRequestDto',
|
||||
'BulkResetTrafficUsersRequestDto': 'BulkUuidsRequestDto',
|
||||
'BulkRevokeUsersSubscriptionRequestDto': 'BulkUuidsRequestDto',
|
||||
|
||||
# Group 6: Hosts Response (6 duplicates)
|
||||
'BulkDeleteHostsResponseDto': 'GetAllHostsResponseDto',
|
||||
'BulkDisableHostsResponseDto': 'GetAllHostsResponseDto',
|
||||
'BulkEnableHostsResponseDto': 'GetAllHostsResponseDto',
|
||||
'SetInboundToManyHostsResponseDto': 'GetAllHostsResponseDto',
|
||||
'SetPortToManyHostsResponseDto': 'GetAllHostsResponseDto',
|
||||
|
||||
# Group 7: Token Responses (5 duplicates)
|
||||
'OAuth2CallbackResponseDto': 'LoginResponseDto',
|
||||
'RegisterResponseDto': 'LoginResponseDto',
|
||||
'TelegramCallbackResponseDto': 'LoginResponseDto',
|
||||
'VerifyPasskeyAuthenticationResponseDto': 'LoginResponseDto',
|
||||
|
||||
# Group 8: Node Responses (5 duplicates)
|
||||
'DisableNodeResponseDto': 'CreateNodeResponseDto',
|
||||
'EnableNodeResponseDto': 'CreateNodeResponseDto',
|
||||
'GetOneNodeResponseDto': 'CreateNodeResponseDto',
|
||||
'UpdateNodeResponseDto': 'CreateNodeResponseDto',
|
||||
|
||||
# Group 9: Empty Wrapper (4 duplicates)
|
||||
'GetPasskeyAuthenticationOptionsResponseDto': 'GetPasskeyRegistrationOptionsResponseDto',
|
||||
'VerifyPasskeyAuthenticationRequestDto': 'GetPasskeyRegistrationOptionsResponseDto',
|
||||
'VerifyPasskeyRegistrationRequestDto': 'GetPasskeyRegistrationOptionsResponseDto',
|
||||
|
||||
# Group 10: Subscription Info (4 duplicates)
|
||||
'GetSubscriptionByShortUuidProtectedResponseDto': 'GetSubscriptionInfoResponseDto',
|
||||
'GetSubscriptionByUsernameResponseDto': 'GetSubscriptionInfoResponseDto',
|
||||
'GetSubscriptionByUuidResponseDto': 'GetSubscriptionInfoResponseDto',
|
||||
|
||||
# Group 11: Snippet Operations (4 duplicates)
|
||||
'CreateSnippetResponseDto': 'GetSnippetsResponseDto',
|
||||
'DeleteSnippetResponseDto': 'GetSnippetsResponseDto',
|
||||
'UpdateSnippetResponseDto': 'GetSnippetsResponseDto',
|
||||
|
||||
# Group 12: HWID Devices (4 duplicates)
|
||||
'CreateUserHwidDeviceResponseDto': 'GetUserHwidDevicesResponseDto',
|
||||
'DeleteAllUserHwidDevicesResponseDto': 'GetUserHwidDevicesResponseDto',
|
||||
'DeleteUserHwidDeviceResponseDto': 'GetUserHwidDevicesResponseDto',
|
||||
|
||||
# Group 13: Billing Nodes (4 duplicates)
|
||||
'CreateInfraBillingNodeResponseDto': 'GetInfraBillingNodesResponseDto',
|
||||
'DeleteInfraBillingNodeByUuidResponseDto': 'GetInfraBillingNodesResponseDto',
|
||||
'UpdateInfraBillingNodeResponseDto': 'GetInfraBillingNodesResponseDto',
|
||||
|
||||
# Group 14: User Search (3 duplicates)
|
||||
'GetUserByEmailResponseDto': 'GetUserByTelegramIdResponseDto',
|
||||
'GetUserByTagResponseDto': 'GetUserByTelegramIdResponseDto',
|
||||
|
||||
# Group 15: Templates (3 duplicates)
|
||||
'CreateSubscriptionTemplateResponseDto': 'GetTemplateResponseDto',
|
||||
'UpdateTemplateResponseDto': 'GetTemplateResponseDto',
|
||||
|
||||
# Group 16: Config Profiles (3 duplicates)
|
||||
'CreateConfigProfileResponseDto': 'GetConfigProfileByUuidResponseDto',
|
||||
'UpdateConfigProfileResponseDto': 'GetConfigProfileByUuidResponseDto',
|
||||
|
||||
# Group 17: Internal Squads (3 duplicates)
|
||||
'CreateInternalSquadResponseDto': 'GetInternalSquadByUuidResponseDto',
|
||||
'UpdateInternalSquadResponseDto': 'GetInternalSquadByUuidResponseDto',
|
||||
|
||||
# Group 18: External Squads (3 duplicates)
|
||||
'CreateExternalSquadResponseDto': 'GetExternalSquadByUuidResponseDto',
|
||||
'UpdateExternalSquadResponseDto': 'GetExternalSquadByUuidResponseDto',
|
||||
|
||||
# Group 19: Hosts (3 duplicates)
|
||||
'CreateHostResponseDto': 'GetOneHostResponseDto',
|
||||
'UpdateHostResponseDto': 'GetOneHostResponseDto',
|
||||
|
||||
# Group 20: Infrastructure Providers (3 duplicates)
|
||||
'CreateInfraProviderResponseDto': 'GetInfraProviderByUuidResponseDto',
|
||||
'UpdateInfraProviderResponseDto': 'GetInfraProviderByUuidResponseDto',
|
||||
|
||||
# Group 21: Billing History (3 duplicates)
|
||||
'CreateInfraBillingHistoryRecordResponseDto': 'GetInfraBillingHistoryRecordsResponseDto',
|
||||
'DeleteInfraBillingHistoryRecordByUuidResponseDto': 'GetInfraBillingHistoryRecordsResponseDto',
|
||||
|
||||
# Group 22: Settings (2 duplicates)
|
||||
'UpdateRemnawaveSettingsResponseDto': 'GetRemnawaveSettingsResponseDto',
|
||||
|
||||
# Group 23: Passkeys (2 duplicates)
|
||||
'DeletePasskeyResponseDto': 'GetAllPasskeysResponseDto',
|
||||
|
||||
# Group 24: Tags (2 duplicates)
|
||||
'GetAllHostTagsResponseDto': 'GetAllTagsResponseDto',
|
||||
|
||||
# Group 25: Inbounds (2 duplicates)
|
||||
'GetInboundsByProfileUuidResponseDto': 'GetAllInboundsResponseDto',
|
||||
|
||||
# Group 26: Snippet Requests (2 duplicates)
|
||||
'UpdateSnippetRequestDto': 'CreateSnippetRequestDto',
|
||||
|
||||
# Group 27: Nodes (2 duplicates)
|
||||
'ReorderNodeResponseDto': 'GetAllNodesResponseDto',
|
||||
|
||||
# Group 28: Subscription Settings (2 duplicates)
|
||||
'UpdateSubscriptionSettingsResponseDto': 'GetSubscriptionSettingsResponseDto',
|
||||
}
|
||||
|
||||
|
||||
def create_canonical_schemas(original_schemas: dict, consolidation_map: dict) -> dict:
|
||||
"""
|
||||
Create new schemas dict with canonical names and new generic schemas.
|
||||
"""
|
||||
# Get all canonical names from mapping
|
||||
canonical_names = set(consolidation_map.values())
|
||||
duplicates_to_remove = set(consolidation_map.keys())
|
||||
|
||||
# Keep only canonical schemas
|
||||
new_schemas = {}
|
||||
for name, schema_def in original_schemas.items():
|
||||
if name not in duplicates_to_remove:
|
||||
new_schemas[name] = schema_def
|
||||
|
||||
# Add new generic schemas based on patterns found
|
||||
new_schemas['DeleteResponseDto'] = {
|
||||
"properties": {
|
||||
"response": {
|
||||
"properties": {
|
||||
"isDeleted": {"type": "boolean"}
|
||||
},
|
||||
"required": ["isDeleted"],
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"required": ["response"],
|
||||
"type": "object"
|
||||
}
|
||||
|
||||
new_schemas['EventResponseDto'] = {
|
||||
"properties": {
|
||||
"response": {
|
||||
"properties": {
|
||||
"eventSent": {"type": "boolean"}
|
||||
},
|
||||
"required": ["eventSent"],
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"required": ["response"],
|
||||
"type": "object"
|
||||
}
|
||||
|
||||
new_schemas['BulkActionResponseDto'] = {
|
||||
"properties": {
|
||||
"response": {
|
||||
"properties": {
|
||||
"affectedRows": {"type": "number"}
|
||||
},
|
||||
"required": ["affectedRows"],
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"required": ["response"],
|
||||
"type": "object"
|
||||
}
|
||||
|
||||
new_schemas['BulkUuidsRequestDto'] = {
|
||||
"properties": {
|
||||
"uuids": {
|
||||
"items": {
|
||||
"format": "uuid",
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"required": ["uuids"],
|
||||
"type": "object"
|
||||
}
|
||||
|
||||
return new_schemas
|
||||
|
||||
|
||||
def replace_refs_in_spec(spec: dict, consolidation_map: dict) -> dict:
|
||||
"""Replace all $ref references to consolidated schemas throughout the spec."""
|
||||
|
||||
def replace_in_value(value):
|
||||
if isinstance(value, dict):
|
||||
if '$ref' in value:
|
||||
ref = value['$ref']
|
||||
if ref.startswith('#/components/schemas/'):
|
||||
schema_name = ref.replace('#/components/schemas/', '')
|
||||
if schema_name in consolidation_map:
|
||||
value['$ref'] = f"#/components/schemas/{consolidation_map[schema_name]}"
|
||||
else:
|
||||
for k, v in value.items():
|
||||
value[k] = replace_in_value(v)
|
||||
elif isinstance(value, list):
|
||||
return [replace_in_value(item) for item in value]
|
||||
|
||||
return value
|
||||
|
||||
replace_in_value(spec)
|
||||
return spec
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python3 create_consolidated_schema.py <input_file> [output_file]", file=sys.stderr)
|
||||
print("Example: python3 create_consolidated_schema.py api-2-2-0.json api-2-2-0-consolidated.json", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
input_file = sys.argv[1]
|
||||
output_file = sys.argv[2] if len(sys.argv) > 2 else input_file.replace('.json', '-consolidated.json')
|
||||
|
||||
try:
|
||||
print(f"📂 Extracting schemas from: {input_file}")
|
||||
original_schemas = extract_schemas_section(input_file)
|
||||
print(f"✓ Found {len(original_schemas)} schemas")
|
||||
|
||||
print("\n🔍 Creating consolidation mapping...")
|
||||
consolidation_map = create_consolidation_map()
|
||||
|
||||
# Count consolidations
|
||||
consolidations = len(consolidation_map)
|
||||
canonical_count = len(set(consolidation_map.values()))
|
||||
|
||||
print(f"✓ Will consolidate {consolidations} schemas into {canonical_count} canonical schemas")
|
||||
|
||||
print("\n📝 Creating consolidated schemas...")
|
||||
new_schemas = create_canonical_schemas(original_schemas, consolidation_map)
|
||||
print(f"✓ New schema count: {len(new_schemas)}")
|
||||
|
||||
print("\n📖 Loading full OpenAPI spec...")
|
||||
with open(input_file, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
|
||||
# Find last valid JSON brace
|
||||
brace_count = 0
|
||||
in_string = False
|
||||
escape = False
|
||||
last_valid = 0
|
||||
|
||||
for i, char in enumerate(content):
|
||||
if escape:
|
||||
escape = False
|
||||
continue
|
||||
if char == '\\':
|
||||
escape = True
|
||||
continue
|
||||
if char == '"':
|
||||
in_string = not in_string
|
||||
continue
|
||||
if not in_string:
|
||||
if char == '{':
|
||||
brace_count += 1
|
||||
elif char == '}':
|
||||
brace_count -= 1
|
||||
if brace_count == 0:
|
||||
last_valid = i + 1
|
||||
|
||||
# Load truncated JSON
|
||||
full_spec = json.loads(content[:last_valid])
|
||||
|
||||
print("🔄 Replacing all schema references...")
|
||||
full_spec = replace_refs_in_spec(full_spec, consolidation_map)
|
||||
|
||||
print("📝 Updating schemas in spec...")
|
||||
full_spec['components']['schemas'] = new_schemas
|
||||
|
||||
print(f"\n💾 Writing consolidated spec to: {output_file}")
|
||||
with open(output_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(full_spec, f, indent=2, ensure_ascii=False)
|
||||
|
||||
# Print summary
|
||||
schemas_removed = len(original_schemas) - len(new_schemas)
|
||||
reduction_pct = (schemas_removed / len(original_schemas)) * 100
|
||||
|
||||
print(f"\n✅ CONSOLIDATION COMPLETE")
|
||||
print(f" Original schemas: {len(original_schemas)}")
|
||||
print(f" Consolidated schemas: {len(new_schemas)}")
|
||||
print(f" Schemas removed: {schemas_removed}")
|
||||
print(f" Reduction: {reduction_pct:.1f}%")
|
||||
print(f" Generic schemas: {canonical_count}")
|
||||
|
||||
# Print mapping summary
|
||||
print(f"\n📋 CONSOLIDATION MAPPING:")
|
||||
grouped = defaultdict(list)
|
||||
for old, new in sorted(consolidation_map.items()):
|
||||
grouped[new].append(old)
|
||||
|
||||
for canonical, duplicates in sorted(grouped.items()):
|
||||
print(f" ✓ {canonical} (← {len(duplicates)} schemas)")
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n✗ Error: {e}", file=sys.stderr)
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+304
@@ -0,0 +1,304 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
OpenAPI Schema Duplicate Finder
|
||||
|
||||
This script analyzes OpenAPI/Swagger JSON files to find duplicate or identical
|
||||
request/response models (DTOs). Useful for identifying opportunities to consolidate
|
||||
schemas and reduce API specification redundancy.
|
||||
|
||||
Usage:
|
||||
python3 find_duplicate_schemas.py <path_to_openapi.json>
|
||||
python3 find_duplicate_schemas.py api-2-2-2.json
|
||||
python3 find_duplicate_schemas.py api-2-2-0.json
|
||||
|
||||
Features:
|
||||
- Handles malformed JSON files by attempting salvage through truncation
|
||||
- Groups identical schemas together
|
||||
- Shows detailed analysis of each duplicate group
|
||||
- Outputs statistics and recommendations
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def find_schemas_section(content: str) -> tuple[str, int]:
|
||||
"""
|
||||
Extract the schemas JSON object from OpenAPI file.
|
||||
|
||||
Returns:
|
||||
Tuple of (schemas_json_string, end_position)
|
||||
"""
|
||||
schemas_start = content.find('"schemas": {')
|
||||
if schemas_start < 0:
|
||||
raise ValueError('Could not find "schemas" section in JSON file')
|
||||
|
||||
schemas_part = content[schemas_start + len('"schemas": '):]
|
||||
|
||||
# Count braces to find the end of the schemas object
|
||||
brace_count = 0
|
||||
in_string = False
|
||||
escape = False
|
||||
end_pos = 0
|
||||
|
||||
for i, char in enumerate(schemas_part):
|
||||
if escape:
|
||||
escape = False
|
||||
continue
|
||||
if char == '\\':
|
||||
escape = True
|
||||
continue
|
||||
if char == '"':
|
||||
in_string = not in_string
|
||||
continue
|
||||
if not in_string:
|
||||
if char == '{':
|
||||
brace_count += 1
|
||||
elif char == '}':
|
||||
brace_count -= 1
|
||||
if brace_count == 0:
|
||||
end_pos = i + 1
|
||||
break
|
||||
|
||||
if end_pos == 0:
|
||||
raise ValueError('Could not find end of schemas section')
|
||||
|
||||
schemas_json = schemas_part[:end_pos]
|
||||
return schemas_json, schemas_start + len('"schemas": ') + end_pos
|
||||
|
||||
|
||||
def load_schemas(filepath: str) -> dict:
|
||||
"""
|
||||
Load schemas from an OpenAPI JSON file.
|
||||
Attempts to handle malformed files by extracting only the schemas section.
|
||||
"""
|
||||
filepath = Path(filepath)
|
||||
if not filepath.exists():
|
||||
raise FileNotFoundError(f"File not found: {filepath}")
|
||||
|
||||
print(f"📂 Reading file: {filepath}", file=sys.stderr)
|
||||
|
||||
with open(filepath, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
|
||||
file_size_mb = len(content) / (1024 * 1024)
|
||||
print(f"📊 File size: {file_size_mb:.2f} MB", file=sys.stderr)
|
||||
|
||||
try:
|
||||
# Try parsing the entire file first
|
||||
spec = json.loads(content)
|
||||
schemas = spec.get('components', {}).get('schemas', {})
|
||||
print(f"✓ File parsed successfully (full JSON)", file=sys.stderr)
|
||||
except json.JSONDecodeError:
|
||||
print(f"⚠ Full JSON parse failed, attempting schema extraction...", file=sys.stderr)
|
||||
|
||||
try:
|
||||
schemas_json, _ = find_schemas_section(content)
|
||||
wrapped = '{"schemas": ' + schemas_json + '}'
|
||||
data = json.loads(wrapped)
|
||||
schemas = data['schemas']
|
||||
print(f"✓ Schemas extracted successfully (partial extraction)", file=sys.stderr)
|
||||
except (ValueError, json.JSONDecodeError) as e:
|
||||
print(f"✗ Error: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
return schemas
|
||||
|
||||
|
||||
def find_duplicates(schemas: dict) -> tuple[dict, list]:
|
||||
"""
|
||||
Find duplicate/identical schemas.
|
||||
|
||||
Returns:
|
||||
Tuple of (schema_groups_dict, sorted_duplicates_list)
|
||||
"""
|
||||
schema_groups = defaultdict(list)
|
||||
|
||||
for name, schema_def in schemas.items():
|
||||
# Convert schema to JSON string for comparison
|
||||
key = json.dumps(schema_def, sort_keys=True, default=str)
|
||||
schema_groups[key].append(name)
|
||||
|
||||
# Extract duplicates (groups with more than one schema)
|
||||
duplicates = sorted(
|
||||
[(v, k) for k, v in schema_groups.items() if len(v) > 1],
|
||||
key=lambda x: len(x[0]),
|
||||
reverse=True
|
||||
)
|
||||
|
||||
return schema_groups, duplicates
|
||||
|
||||
|
||||
def print_summary(schemas: dict, schema_groups: dict, duplicates: list) -> None:
|
||||
"""Print summary statistics."""
|
||||
print("\n" + "=" * 130)
|
||||
print("📈 SUMMARY STATISTICS")
|
||||
print("=" * 130)
|
||||
print(f"Total schemas: {len(schemas)}")
|
||||
print(f"Unique definitions: {len(schema_groups)}")
|
||||
print(f"Duplicate groups: {len(duplicates)}")
|
||||
print(f"Redundant schemas: {len(schemas) - len(schema_groups)}")
|
||||
print("=" * 130 + "\n")
|
||||
|
||||
|
||||
def print_duplicates(duplicates: list, max_groups: int = None) -> None:
|
||||
"""Print detailed information about each duplicate group."""
|
||||
if not duplicates:
|
||||
print("✓ No duplicate schemas found - all schemas are unique!")
|
||||
return
|
||||
|
||||
print("=" * 130)
|
||||
print("🔍 DUPLICATE SCHEMAS FOUND:")
|
||||
print("=" * 130)
|
||||
|
||||
for idx, (names, schema_json) in enumerate(duplicates[:max_groups] if max_groups else duplicates, 1):
|
||||
schema_def = json.loads(schema_json)
|
||||
|
||||
print(f"\n[GROUP {idx}] {len(names)} IDENTICAL MODELS")
|
||||
print(f"Models: {', '.join(sorted(names))}")
|
||||
|
||||
# Show schema structure details
|
||||
print(f"\nSchema Type Details:")
|
||||
if schema_def.get('type') == 'object':
|
||||
if 'properties' in schema_def:
|
||||
props = list(schema_def['properties'].keys())
|
||||
print(f" • Object with {len(props)} properties")
|
||||
print(f" • Fields: {props[:8]}", end='')
|
||||
if len(props) > 8:
|
||||
print(f" ... (+{len(props)-8} more)")
|
||||
else:
|
||||
print()
|
||||
if 'required' in schema_def:
|
||||
print(f" • Required: {schema_def['required']}")
|
||||
elif '$ref' in schema_def:
|
||||
print(f" • Reference: {schema_def['$ref']}")
|
||||
else:
|
||||
print(f" • Type: {schema_def.get('type', 'unknown')}")
|
||||
|
||||
# Show schema definition preview
|
||||
schema_preview = json.dumps(schema_def, indent=2)
|
||||
lines = schema_preview.split('\n')[:12]
|
||||
print(f"\nSchema Definition (preview):")
|
||||
for line in lines:
|
||||
print(f" {line}")
|
||||
if len(schema_preview.split('\n')) > 12:
|
||||
print(f" ... ({len(schema_preview.split('\n')) - 12} more lines)")
|
||||
|
||||
print("-" * 130)
|
||||
|
||||
|
||||
def print_recommendations(duplicates: list) -> None:
|
||||
"""Print consolidation recommendations."""
|
||||
if not duplicates:
|
||||
return
|
||||
|
||||
print("\n" + "=" * 130)
|
||||
print("💡 CONSOLIDATION RECOMMENDATIONS")
|
||||
print("=" * 130)
|
||||
|
||||
# Categorize by group size
|
||||
large_groups = [d for d in duplicates if len(d[0]) >= 5]
|
||||
medium_groups = [d for d in duplicates if 3 <= len(d[0]) < 5]
|
||||
small_groups = [d for d in duplicates if len(d[0]) == 2]
|
||||
|
||||
if large_groups:
|
||||
print(f"\n🔴 HIGH PRIORITY (5+ duplicates):")
|
||||
for names, _ in large_groups:
|
||||
print(f" • {len(names)} models can be consolidated: {names[0]}* (and {len(names)-1} others)")
|
||||
|
||||
if medium_groups:
|
||||
print(f"\n🟡 MEDIUM PRIORITY (3-4 duplicates):")
|
||||
for names, _ in medium_groups:
|
||||
print(f" • {len(names)} models: {', '.join(names[:2])}...")
|
||||
|
||||
if small_groups:
|
||||
print(f"\n🟢 LOW PRIORITY (2 duplicates):")
|
||||
total_pairs = len(small_groups)
|
||||
print(f" • {total_pairs} pairs of duplicate schemas")
|
||||
|
||||
print("\n" + "=" * 130)
|
||||
|
||||
|
||||
def print_grouped_by_pattern(duplicates: list) -> None:
|
||||
"""Print duplicates grouped by response pattern."""
|
||||
if not duplicates:
|
||||
return
|
||||
|
||||
print("\n" + "=" * 130)
|
||||
print("🎯 PATTERNS IDENTIFIED")
|
||||
print("=" * 130)
|
||||
|
||||
patterns = {
|
||||
"Delete Operations": [],
|
||||
"Empty Wrapper": [],
|
||||
"Event Based": [],
|
||||
"Bulk Operations": [],
|
||||
"Token Responses": [],
|
||||
"List Responses": [],
|
||||
"Other": []
|
||||
}
|
||||
|
||||
for names, schema_json in duplicates:
|
||||
schema_def = json.loads(schema_json)
|
||||
|
||||
# Categorize by pattern
|
||||
if isinstance(schema_def.get('properties', {}).get('response'), dict):
|
||||
resp = schema_def['properties']['response']
|
||||
if resp.get('properties', {}).get('isDeleted'):
|
||||
patterns["Delete Operations"].append((len(names), names))
|
||||
elif resp.get('properties', {}).get('eventSent'):
|
||||
patterns["Event Based"].append((len(names), names))
|
||||
elif resp.get('properties', {}).get('affectedRows'):
|
||||
patterns["Bulk Operations"].append((len(names), names))
|
||||
elif resp.get('properties', {}).get('accessToken'):
|
||||
patterns["Token Responses"].append((len(names), names))
|
||||
elif not resp.get('properties'):
|
||||
patterns["Empty Wrapper"].append((len(names), names))
|
||||
elif isinstance(resp.get('items'), dict):
|
||||
patterns["List Responses"].append((len(names), names))
|
||||
else:
|
||||
patterns["Other"].append((len(names), names))
|
||||
else:
|
||||
patterns["Other"].append((len(names), names))
|
||||
|
||||
for pattern_name, items in patterns.items():
|
||||
if items:
|
||||
total_models = sum(count for count, _ in items)
|
||||
print(f"\n{pattern_name}: {total_models} total models across {len(items)} groups")
|
||||
for count, names in sorted(items, key=lambda x: x[0], reverse=True):
|
||||
print(f" [{count}] {', '.join(names[:3])}{'...' if len(names) > 3 else ''}")
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python3 find_duplicate_schemas.py <openapi_file.json>", file=sys.stderr)
|
||||
print("\nExamples:", file=sys.stderr)
|
||||
print(" python3 find_duplicate_schemas.py api-2-2-2.json", file=sys.stderr)
|
||||
print(" python3 find_duplicate_schemas.py openapi.json", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
filepath = sys.argv[1]
|
||||
max_groups = int(sys.argv[2]) if len(sys.argv) > 2 else None
|
||||
|
||||
try:
|
||||
# Load schemas
|
||||
schemas = load_schemas(filepath)
|
||||
|
||||
# Find duplicates
|
||||
schema_groups, duplicates = find_duplicates(schemas)
|
||||
|
||||
# Print results
|
||||
print_summary(schemas, schema_groups, duplicates)
|
||||
print_duplicates(duplicates, max_groups)
|
||||
print_recommendations(duplicates)
|
||||
print_grouped_by_pattern(duplicates)
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n✗ Error: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+179
@@ -0,0 +1,179 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Generate complete client_ext.go with all API operations organized by controller.
|
||||
|
||||
This script parses the OpenAPI spec and generates organized sub-clients for all API endpoints.
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def to_camel_case(snake_str):
|
||||
"""Convert snake_case to camelCase"""
|
||||
components = snake_str.split('_')
|
||||
return components[0] + ''.join(x.title() for x in components[1:])
|
||||
|
||||
|
||||
def get_method_name(operation_id):
|
||||
"""Extract method name from operationId like 'UsersController_createUser' -> 'CreateUser'"""
|
||||
parts = operation_id.split('_')
|
||||
if len(parts) >= 2:
|
||||
return to_camel_case('_'.join(parts[1:]))
|
||||
return operation_id
|
||||
|
||||
|
||||
def get_client_name(tag):
|
||||
"""Convert tag to client name like 'Users Controller' -> 'UsersClient'"""
|
||||
# Remove brackets, clean up
|
||||
clean_tag = tag.replace('[', '').replace(']', '').replace(' ', '_').replace('-', '_')
|
||||
words = clean_tag.split('_')
|
||||
return ''.join(w.title() for w in words if w) + 'Client'
|
||||
|
||||
|
||||
def get_field_name(tag):
|
||||
"""Convert tag to field name like 'Users Controller' -> 'users'"""
|
||||
clean_tag = tag.replace('[', '').replace(']', '').replace(' ', '_').replace('-', '_')
|
||||
words = clean_tag.split('_')
|
||||
name = ''.join(w.title() for w in words if w)
|
||||
return name[0].lower() + name[1:] if name else 'client'
|
||||
|
||||
|
||||
def parse_operation_id(operation_id):
|
||||
"""Parse operation ID to extract controller and method parts"""
|
||||
parts = operation_id.split('_')
|
||||
return parts[0] if parts else '', '_'.join(parts[1:]) if len(parts) > 1 else ''
|
||||
|
||||
|
||||
def generate_client_method(operation_id, op_details):
|
||||
"""Generate a method signature for the operation"""
|
||||
method_name = get_method_name(operation_id)
|
||||
|
||||
# Determine parameters
|
||||
params_part = ""
|
||||
return_type = "error"
|
||||
|
||||
# Check for parameters
|
||||
if op_details.get('params'):
|
||||
params_part = f"params {operation_id.split('_')[0]}*"
|
||||
|
||||
# Check for request body
|
||||
if op_details.get('requestBody'):
|
||||
if params_part:
|
||||
params_part += ", "
|
||||
request_type = f"*{operation_id.split('_')[0]}"
|
||||
params_part += f"request {request_type}"
|
||||
|
||||
# Simple method - delegate to base client
|
||||
return f"""func ({get_field_name("dummy")[0]}c *{get_client_name("dummy")}) {method_name}(ctx context.Context{", " + params_part if params_part else ""}) error {{
|
||||
\treturn nil // Implementation delegated to base Client
|
||||
}}"""
|
||||
|
||||
|
||||
def main():
|
||||
with open('api-2-2-2-consolidated.json', 'r') as f:
|
||||
spec = json.load(f)
|
||||
|
||||
paths = spec.get('paths', {})
|
||||
operations_by_controller = {}
|
||||
|
||||
# Group operations by controller
|
||||
for path, methods in paths.items():
|
||||
for method, details in methods.items():
|
||||
if isinstance(details, dict) and 'operationId' in details:
|
||||
tag = details.get('tags', ['Unknown'])[0]
|
||||
|
||||
if tag not in operations_by_controller:
|
||||
operations_by_controller[tag] = []
|
||||
|
||||
operations_by_controller[tag].append({
|
||||
'operationId': details['operationId'],
|
||||
'method': method.upper(),
|
||||
'path': path,
|
||||
})
|
||||
|
||||
# Generate client_ext.go content
|
||||
content = '''// Code generated by client_ext generator. DO NOT EDIT manually.
|
||||
// This file extends the base Client with organized sub-client access patterns for all API operations.
|
||||
|
||||
package api
|
||||
|
||||
import "context"
|
||||
|
||||
// ClientExt wraps the base Client and adds organized sub-client methods.
|
||||
type ClientExt struct {
|
||||
\t*Client
|
||||
'''
|
||||
|
||||
# Add fields
|
||||
field_names = set()
|
||||
for tag in sorted(operations_by_controller.keys()):
|
||||
field_name = get_field_name(tag)
|
||||
if field_name not in field_names:
|
||||
field_names.add(field_name)
|
||||
content += f'\t{field_name} *{get_client_name(tag)}\n'
|
||||
|
||||
content += '''}
|
||||
|
||||
// NewClientExt wraps an existing Client with sub-client access.
|
||||
func NewClientExt(client *Client) *ClientExt {
|
||||
\treturn &ClientExt{
|
||||
\t\tClient: client,
|
||||
'''
|
||||
|
||||
# Add initializations
|
||||
for tag in sorted(operations_by_controller.keys()):
|
||||
field_name = get_field_name(tag)
|
||||
client_name = get_client_name(tag)
|
||||
content += f'\t\t{field_name}: New{client_name}(client),\n'
|
||||
|
||||
content += '''\t}
|
||||
}
|
||||
|
||||
'''
|
||||
|
||||
# Add accessor methods
|
||||
for tag in sorted(operations_by_controller.keys()):
|
||||
field_name = get_field_name(tag)
|
||||
client_name = get_client_name(tag)
|
||||
content += f'func (c *ClientExt) {to_camel_case(field_name)}() *{client_name} {{ return c.{field_name} }}\n'
|
||||
|
||||
content += '\n'
|
||||
|
||||
# Generate sub-client types and methods
|
||||
for tag in sorted(operations_by_controller.keys()):
|
||||
client_name = get_client_name(tag)
|
||||
field_name = get_field_name(tag)
|
||||
operations = operations_by_controller[tag]
|
||||
|
||||
content += f'''
|
||||
// {client_name} provides organized access to {tag.lower()} operations
|
||||
type {client_name} struct{{ client *Client }}
|
||||
func New{client_name}(c *Client) *{client_name} {{ return &{client_name}{{client: c}} }}
|
||||
|
||||
'''
|
||||
|
||||
# Generate methods for this controller
|
||||
for op in operations:
|
||||
op_id = op['operationId']
|
||||
method_name = get_method_name(op_id)
|
||||
|
||||
# Simplified approach - just delegate to base client
|
||||
content += f"func ({field_name[0]}c *{client_name}) {method_name}(ctx context.Context) error {{\n"
|
||||
content += f"\t// Delegate to base client method\n"
|
||||
content += f"\treturn nil\n"
|
||||
content += f"}}\n\n"
|
||||
|
||||
# Write to file
|
||||
output_path = Path('api/client_ext.go')
|
||||
with open(output_path, 'w') as f:
|
||||
f.write(content)
|
||||
|
||||
print(f"✅ Generated {output_path}")
|
||||
print(f" {len(operations_by_controller)} controllers")
|
||||
print(f" {sum(len(ops) for ops in operations_by_controller.values())} total operations")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+234
@@ -0,0 +1,234 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Final client_ext.go generator that actually works.
|
||||
Reads api-2-2-2-consolidated.json and oas_client_gen.go
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
|
||||
print("=" * 70)
|
||||
print("CLIENT_EXT.GO GENERATOR")
|
||||
print("=" * 70)
|
||||
|
||||
# Step 1: Parse oas_client_gen.go for method signatures
|
||||
print("\n[1/4] Parsing oas_client_gen.go...")
|
||||
with open('api/oas_client_gen.go', 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
# Extract method signatures more carefully
|
||||
methods = {}
|
||||
# Match: func (c *Client) MethodName(ctx context.Context, ...) (...) {
|
||||
pattern = r'func \(c \*Client\) (\w+)\((ctx context\.Context(?:,\s*[^)]+)?)\)\s*\(([^)]+)\)'
|
||||
|
||||
for match in re.finditer(pattern, content, re.MULTILINE):
|
||||
method_name = match.group(1)
|
||||
if method_name in ['requestURL', 'sendApiTokensControllerCreate']: # Skip internal
|
||||
continue
|
||||
if method_name.startswith('send'):
|
||||
continue
|
||||
|
||||
full_params = match.group(2) # "ctx context.Context, request *Type, params ParamsType"
|
||||
returns = match.group(3) # "TypeRes, error"
|
||||
|
||||
# Parse params (skip ctx)
|
||||
params_list = []
|
||||
if ', ' in full_params:
|
||||
params_str = full_params.split(', ', 1)[1] # Remove "ctx context.Context"
|
||||
# Split remaining params carefully
|
||||
for param in re.findall(r'(\w+)\s+([\*\w\.]+)', params_str):
|
||||
params_list.append((param[0], param[1]))
|
||||
|
||||
# Parse returns
|
||||
returns_list = [r.strip() for r in returns.split(',')]
|
||||
|
||||
methods[method_name] = {
|
||||
'params': params_list,
|
||||
'returns': returns_list
|
||||
}
|
||||
|
||||
print(f" ✓ Found {len(methods)} client methods")
|
||||
|
||||
# Step 2: Parse api-2-2-2-consolidated.json for operations
|
||||
print("\n[2/4] Parsing api-2-2-2-consolidated.json...")
|
||||
with open('api-2-2-2-consolidated.json', 'r') as f:
|
||||
spec = json.load(f)
|
||||
|
||||
operations_by_controller = {}
|
||||
|
||||
for path, path_item in spec.get('paths', {}).items():
|
||||
for http_method, op_spec in path_item.items():
|
||||
if http_method not in ['get', 'post', 'put', 'patch', 'delete']:
|
||||
continue
|
||||
|
||||
op_id = op_spec.get('operationId')
|
||||
if not op_id or '_' not in op_id:
|
||||
continue
|
||||
|
||||
# Parse: "ApiTokensController_create" -> controller="ApiTokensController", method="create"
|
||||
parts = op_id.split('_', 1)
|
||||
controller_full = parts[0] # e.g., "ApiTokensController"
|
||||
method_snake = parts[1] # e.g., "create"
|
||||
|
||||
# Controller name without "Controller" suffix
|
||||
controller = controller_full.replace('Controller', '')
|
||||
|
||||
# Convert method to PascalCase: findAll -> FindAll, get_status -> GetStatus
|
||||
# Just capitalize first letter of each word, preserve rest
|
||||
def to_pascal(s):
|
||||
if not s:
|
||||
return s
|
||||
# Capitalize first letter, keep rest as-is
|
||||
return s[0].upper() + s[1:]
|
||||
|
||||
# Split by underscore and capitalize each part
|
||||
parts = method_snake.split('_')
|
||||
method_pascal = ''.join(to_pascal(p) for p in parts)
|
||||
|
||||
# The actual Go method name in oas_client_gen.go
|
||||
go_method = controller_full + method_pascal # e.g., "ApiTokensControllerCreate"
|
||||
|
||||
if controller not in operations_by_controller:
|
||||
operations_by_controller[controller] = []
|
||||
|
||||
operations_by_controller[controller].append({
|
||||
'operationId': op_id,
|
||||
'goMethod': go_method,
|
||||
'displayMethod': method_pascal
|
||||
})
|
||||
|
||||
total_ops = sum(len(ops) for ops in operations_by_controller.values())
|
||||
print(f" ✓ Found {total_ops} operations in {len(operations_by_controller)} controllers")
|
||||
|
||||
# Step 3: Generate code
|
||||
print("\n[3/4] Generating code...")
|
||||
|
||||
def to_camel(s):
|
||||
"""PascalCase -> camelCase"""
|
||||
return s[0].lower() + s[1:] if s else s
|
||||
|
||||
code = '''// Code generated by generate_clientext_final.py. DO NOT EDIT manually.
|
||||
// Generated from api-2-2-2-consolidated.json with renamed schemas.
|
||||
|
||||
package api
|
||||
|
||||
import "context"
|
||||
|
||||
// ClientExt wraps the base Client with organized sub-client access.
|
||||
type ClientExt struct {
|
||||
\t*Client
|
||||
'''
|
||||
|
||||
# Add fields for each controller
|
||||
for controller in sorted(operations_by_controller.keys()):
|
||||
field_name = to_camel(controller)
|
||||
code += f'\t{field_name} *{controller}Client\n'
|
||||
|
||||
code += '''}
|
||||
|
||||
// NewClientExt creates a new ClientExt wrapper.
|
||||
func NewClientExt(client *Client) *ClientExt {
|
||||
\treturn &ClientExt{
|
||||
\t\tClient: client,
|
||||
'''
|
||||
|
||||
# Initialize fields
|
||||
for controller in sorted(operations_by_controller.keys()):
|
||||
field_name = to_camel(controller)
|
||||
code += f'\t\t{field_name}: New{controller}Client(client),\n'
|
||||
|
||||
code += '''\t}
|
||||
}
|
||||
|
||||
'''
|
||||
|
||||
# Accessor methods
|
||||
for controller in sorted(operations_by_controller.keys()):
|
||||
field_name = to_camel(controller)
|
||||
code += f'''// {controller} returns the {controller}Client.
|
||||
func (ce *ClientExt) {controller}() *{controller}Client {{
|
||||
\treturn ce.{field_name}
|
||||
}}
|
||||
|
||||
'''
|
||||
|
||||
# Sub-client types and methods
|
||||
for controller in sorted(operations_by_controller.keys()):
|
||||
code += f'''// {controller}Client provides {controller} operations.
|
||||
type {controller}Client struct {{
|
||||
\tclient *Client
|
||||
}}
|
||||
|
||||
// New{controller}Client creates a new {controller}Client.
|
||||
func New{controller}Client(client *Client) *{controller}Client {{
|
||||
\treturn &{controller}Client{{client: client}}
|
||||
}}
|
||||
|
||||
'''
|
||||
|
||||
# Generate methods for this controller
|
||||
for op in sorted(operations_by_controller[controller], key=lambda x: x['goMethod']):
|
||||
go_method = op['goMethod']
|
||||
display_method = op['displayMethod']
|
||||
op_id = op['operationId']
|
||||
|
||||
if go_method not in methods:
|
||||
print(f" ⚠ Warning: {go_method} not found in oas_client_gen.go")
|
||||
continue
|
||||
|
||||
method_info = methods[go_method]
|
||||
params = method_info['params']
|
||||
returns = method_info['returns']
|
||||
|
||||
# Build parameter list
|
||||
if params:
|
||||
params_sig = ', '.join([f'{p[0]} {p[1]}' for p in params])
|
||||
params_call = ', '.join([p[0] for p in params])
|
||||
else:
|
||||
params_sig = ''
|
||||
params_call = ''
|
||||
|
||||
# Build return type
|
||||
if returns:
|
||||
ret_type = ', '.join(returns)
|
||||
if len(returns) > 1:
|
||||
ret_type = f'({ret_type})'
|
||||
else:
|
||||
ret_type = ''
|
||||
|
||||
# Generate method
|
||||
code += f'''// {display_method} calls {op_id}.
|
||||
func (sc *{controller}Client) {display_method}(ctx context.Context'''
|
||||
|
||||
if params_sig:
|
||||
code += f', {params_sig}'
|
||||
|
||||
code += ')'
|
||||
|
||||
if ret_type:
|
||||
code += f' {ret_type}'
|
||||
|
||||
code += ' {\n'
|
||||
|
||||
if returns:
|
||||
code += f'\treturn sc.client.{go_method}(ctx'
|
||||
else:
|
||||
code += f'\tsc.client.{go_method}(ctx'
|
||||
|
||||
if params_call:
|
||||
code += f', {params_call}'
|
||||
|
||||
code += ')\n}\n\n'
|
||||
|
||||
# Step 4: Write to file
|
||||
print("\n[4/4] Writing api/client_ext.go...")
|
||||
with open('api/client_ext.go', 'w') as f:
|
||||
f.write(code)
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
print(f"✅ SUCCESS!")
|
||||
print("=" * 70)
|
||||
print(f" Controllers: {len(operations_by_controller)}")
|
||||
print(f" Operations: {total_ops}")
|
||||
print(f" File: api/client_ext.go")
|
||||
print(f" Uses: api-2-2-2-consolidated.json (renamed schemas)")
|
||||
print("=" * 70)
|
||||
Executable
+792
@@ -0,0 +1,792 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Complete API Processing Pipeline
|
||||
=================================
|
||||
|
||||
This script processes OpenAPI specs through the complete workflow:
|
||||
1. Smart consolidate schemas (unify duplicates + error responses)
|
||||
2. Generate Go client via ogen
|
||||
3. Generate client_ext.go wrapper
|
||||
|
||||
Usage:
|
||||
cd /path/to/remnawave-api-go
|
||||
python3 scripts/pipeline.py specs/api-2-3-0.json
|
||||
"""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Tuple
|
||||
|
||||
from smart_consolidate import SmartConsolidator, InlineSchemaExtractor, unify_error_responses, fix_nullable_without_type
|
||||
|
||||
|
||||
class Colors:
|
||||
HEADER = '\033[95m'
|
||||
BLUE = '\033[94m'
|
||||
CYAN = '\033[96m'
|
||||
GREEN = '\033[92m'
|
||||
YELLOW = '\033[93m'
|
||||
RED = '\033[91m'
|
||||
END = '\033[0m'
|
||||
BOLD = '\033[1m'
|
||||
|
||||
|
||||
def print_step(step: int, total: int, title: str):
|
||||
"""Print a step header"""
|
||||
print(f"\n{Colors.BOLD}{Colors.CYAN}{'='*70}")
|
||||
print(f"STEP {step}/{total}: {title}")
|
||||
print(f"{'='*70}{Colors.END}\n")
|
||||
|
||||
|
||||
def print_success(message: str):
|
||||
print(f"{Colors.GREEN}✓ {message}{Colors.END}")
|
||||
|
||||
|
||||
def print_warning(message: str):
|
||||
print(f"{Colors.YELLOW}⚠ {message}{Colors.END}")
|
||||
|
||||
|
||||
def print_error(message: str):
|
||||
print(f"{Colors.RED}✗ {message}{Colors.END}")
|
||||
|
||||
|
||||
def print_info(message: str):
|
||||
print(f"{Colors.BLUE}→ {message}{Colors.END}")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# STEP 1: SMART CONSOLIDATE SCHEMAS
|
||||
# ============================================================================
|
||||
|
||||
def smart_consolidate_schemas(input_file: str, output_file: str, skip_inline_extraction: bool = False) -> Tuple[int, int, dict]:
|
||||
"""
|
||||
Consolidate duplicate schemas using smart analysis.
|
||||
Combines old Steps 1 (consolidate) and 2 (rename) into one step.
|
||||
"""
|
||||
print_info(f"Loading {input_file}...")
|
||||
with open(input_file, 'r') as f:
|
||||
spec = json.load(f)
|
||||
|
||||
original_count = len(spec.get('components', {}).get('schemas', {}))
|
||||
|
||||
print_info("Analyzing schemas with SmartConsolidator...")
|
||||
consolidator = SmartConsolidator(spec)
|
||||
|
||||
# Analyze duplicates
|
||||
report = consolidator.analyze_duplicates()
|
||||
print_info(f"Found {report['exact']['count']} exact duplicate groups ({report['exact']['total_schemas']} schemas)")
|
||||
print_info(f"Found {report['structural']['count']} structural duplicate groups")
|
||||
|
||||
if report['near_duplicates']['count'] > 0:
|
||||
print_warning(f"Found {report['near_duplicates']['count']} near-duplicate groups (metadata differs)")
|
||||
|
||||
if report['constraint_only']['count'] > 0:
|
||||
print_warning(f"Found {report['constraint_only']['count']} constraint-only groups (validation differs)")
|
||||
|
||||
# Consolidate
|
||||
rename_map, stats = consolidator.consolidate()
|
||||
|
||||
if not rename_map:
|
||||
print_warning("No duplicates to consolidate")
|
||||
return original_count, original_count, {}
|
||||
|
||||
# Apply consolidation
|
||||
new_spec = consolidator.apply_consolidation(rename_map)
|
||||
|
||||
# Unify error responses
|
||||
print_info("Unifying error responses...")
|
||||
new_spec, error_stats = unify_error_responses(new_spec)
|
||||
if error_stats['total_replaced'] > 0:
|
||||
print_info(f"Unified {error_stats['total_replaced']} error responses (400: {error_stats['responses_unified'].get('400', 0)}, 401: {error_stats['responses_unified'].get('401', 0)})")
|
||||
stats['unified_errors'] = error_stats['total_replaced']
|
||||
|
||||
# Fix nullable properties without type (ogen requires type for nullable fields)
|
||||
print_info("Fixing nullable properties without type...")
|
||||
new_spec, nullable_fixed = fix_nullable_without_type(new_spec)
|
||||
if nullable_fixed > 0:
|
||||
print_info(f"Fixed {nullable_fixed} nullable properties without type")
|
||||
stats['nullable_fixed'] = nullable_fixed
|
||||
|
||||
# Extract inline schemas for reuse (optional - can cause conflicts in some specs)
|
||||
if not skip_inline_extraction:
|
||||
print_info("Extracting inline schemas for reuse...")
|
||||
extractor = InlineSchemaExtractor(new_spec)
|
||||
new_spec, extract_stats = extractor.extract_inline_schemas()
|
||||
|
||||
if extract_stats['extracted_count'] > 0:
|
||||
print_info(f"Extracted {extract_stats['extracted_count']} inline schemas")
|
||||
stats['extracted_schemas'] = extract_stats['extracted_count']
|
||||
else:
|
||||
print_info("Skipping inline schema extraction")
|
||||
|
||||
print_info(f"Writing {output_file}...")
|
||||
with open(output_file, 'w') as f:
|
||||
json.dump(new_spec, f, indent=2, ensure_ascii=False)
|
||||
|
||||
# Print top consolidated groups
|
||||
print_info("Top consolidated groups:")
|
||||
for name, schemas in sorted(stats['consolidated_names'].items(), key=lambda x: -len(x[1]))[:5]:
|
||||
print(f" {name} <- {len(schemas)} schemas")
|
||||
|
||||
new_count = len(new_spec.get('components', {}).get('schemas', {}))
|
||||
stats['final_count'] = new_count
|
||||
print_success(f"Consolidated {original_count} → {new_count} schemas (-{original_count - new_count}, -{(original_count-new_count)*100//original_count}%)")
|
||||
|
||||
return original_count, new_count, stats
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# STEP 1.5: PATCH SPEC FOR TEXT/PLAIN SUBSCRIPTION ENDPOINTS
|
||||
# ============================================================================
|
||||
|
||||
# These subscription endpoints return text/plain (subscription configs as strings),
|
||||
# but the OpenAPI spec doesn't declare response content, causing ogen to skip them.
|
||||
SUBSCRIPTION_TEXT_OPERATIONS = [
|
||||
'SubscriptionController_getSubscription',
|
||||
'SubscriptionController_getSubscriptionByClientType',
|
||||
'SubscriptionController_getSubscriptionWithType',
|
||||
]
|
||||
|
||||
|
||||
def patch_subscription_text_responses(spec: dict) -> int:
|
||||
"""
|
||||
Patch the spec to add text/plain response content
|
||||
for subscription endpoints that return raw subscription configs.
|
||||
Modifies spec in-place. Returns the number of operations patched.
|
||||
"""
|
||||
patched = 0
|
||||
for path, path_item in spec.get('paths', {}).items():
|
||||
for http_method, op in path_item.items():
|
||||
if not isinstance(op, dict):
|
||||
continue
|
||||
op_id = op.get('operationId', '')
|
||||
if op_id not in SUBSCRIPTION_TEXT_OPERATIONS:
|
||||
continue
|
||||
|
||||
responses = op.get('responses', {})
|
||||
resp_200 = responses.get('200', {})
|
||||
|
||||
# Add text/plain content if not already present
|
||||
if 'content' not in resp_200:
|
||||
resp_200['content'] = {}
|
||||
if 'text/plain' not in resp_200['content']:
|
||||
resp_200['content']['text/plain'] = {
|
||||
'schema': {'type': 'string'}
|
||||
}
|
||||
patched += 1
|
||||
print_info(f"Patched {op_id} with text/plain response")
|
||||
|
||||
responses['200'] = resp_200
|
||||
op['responses'] = responses
|
||||
|
||||
return patched
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# STEP 1.6: SHORTEN OPERATION IDS
|
||||
# ============================================================================
|
||||
|
||||
def shorten_operation_ids(spec: dict) -> int:
|
||||
"""
|
||||
Strip 'Controller' from all operationIds to produce shorter Go type names.
|
||||
E.g. SubscriptionController_getSubscription → Subscription_getSubscription
|
||||
Modifies spec in-place. Returns the number of operations renamed.
|
||||
"""
|
||||
renamed = 0
|
||||
for path, path_item in spec.get('paths', {}).items():
|
||||
for http_method, op in path_item.items():
|
||||
if not isinstance(op, dict):
|
||||
continue
|
||||
op_id = op.get('operationId', '')
|
||||
if 'Controller' in op_id:
|
||||
op['operationId'] = op_id.replace('Controller', '')
|
||||
renamed += 1
|
||||
return renamed
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# STEP 1.7: STRIP 'Dto' SUFFIX FROM SCHEMA NAMES
|
||||
# ============================================================================
|
||||
|
||||
def strip_dto_suffix(spec: dict) -> int:
|
||||
"""
|
||||
Remove 'Dto' suffix from all schema names and update all $ref pointers.
|
||||
E.g. CreateUserRequestDto → CreateUserRequest
|
||||
Modifies spec in-place. Returns the number of schemas renamed.
|
||||
"""
|
||||
schemas = spec.get('components', {}).get('schemas', {})
|
||||
rename_map = {}
|
||||
|
||||
for name in list(schemas.keys()):
|
||||
if name.endswith('Dto'):
|
||||
new_name = name[:-3]
|
||||
# Avoid collision with existing schema
|
||||
if new_name not in schemas and new_name not in rename_map.values():
|
||||
rename_map[name] = new_name
|
||||
|
||||
if not rename_map:
|
||||
return 0
|
||||
|
||||
# Rename schemas
|
||||
new_schemas = {}
|
||||
for name, schema in schemas.items():
|
||||
new_name = rename_map.get(name, name)
|
||||
new_schemas[new_name] = schema
|
||||
spec['components']['schemas'] = new_schemas
|
||||
|
||||
# Update all $ref pointers throughout the spec
|
||||
old_prefix = '#/components/schemas/'
|
||||
ref_map = {f'{old_prefix}{old}': f'{old_prefix}{new}' for old, new in rename_map.items()}
|
||||
|
||||
def _update_refs(obj):
|
||||
if isinstance(obj, dict):
|
||||
if '$ref' in obj and obj['$ref'] in ref_map:
|
||||
obj['$ref'] = ref_map[obj['$ref']]
|
||||
for v in obj.values():
|
||||
_update_refs(v)
|
||||
elif isinstance(obj, list):
|
||||
for item in obj:
|
||||
_update_refs(item)
|
||||
|
||||
_update_refs(spec)
|
||||
return len(rename_map)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# STEP 1.8: FIX NUMERIC QUERY PARAMETERS THAT SHOULD BE INTEGERS
|
||||
# ============================================================================
|
||||
|
||||
# Query parameter names that are semantically integers (pagination, limits, counts)
|
||||
INTEGER_QUERY_PARAMS = {'size', 'start', 'topUsersLimit', 'topNodesLimit', 'limit', 'offset', 'page', 'count'}
|
||||
|
||||
|
||||
def fix_number_query_params(spec: dict) -> int:
|
||||
"""
|
||||
Change query parameters with type 'number' to 'integer' when they represent
|
||||
pagination or limit values. The upstream OpenAPI spec incorrectly uses 'number'
|
||||
for these, which produces float64 in Go instead of int.
|
||||
Modifies spec in-place. Returns the number of parameters fixed.
|
||||
"""
|
||||
fixed = 0
|
||||
for path, path_item in spec.get('paths', {}).items():
|
||||
for http_method, op in path_item.items():
|
||||
if not isinstance(op, dict):
|
||||
continue
|
||||
for param in op.get('parameters', []):
|
||||
if param.get('in') != 'query':
|
||||
continue
|
||||
schema = param.get('schema', {})
|
||||
if schema.get('type') == 'number' and param.get('name') in INTEGER_QUERY_PARAMS:
|
||||
schema['type'] = 'integer'
|
||||
fixed += 1
|
||||
return fixed
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# STEP 2: GENERATE GO CLIENT WITH OGEN
|
||||
# ============================================================================
|
||||
|
||||
def generate_ogen_client(spec_file: str) -> bool:
|
||||
"""Generate Go client using ogen"""
|
||||
print_info(f"Running ogen with {spec_file}...")
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[
|
||||
'go', 'run', 'github.com/ogen-go/ogen/cmd/ogen@v1.19.0',
|
||||
'--config', '.ogen.yml',
|
||||
'--target', 'api',
|
||||
'--package', 'api',
|
||||
'--clean',
|
||||
spec_file
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
print_success(f"Go client generated from {spec_file}")
|
||||
return True
|
||||
else:
|
||||
print_error(f"ogen generation failed: {result.stderr}")
|
||||
return False
|
||||
except subprocess.TimeoutExpired:
|
||||
print_error("ogen generation timed out")
|
||||
return False
|
||||
except Exception as e:
|
||||
print_error(f"Error running ogen: {e}")
|
||||
return False
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# STEP 3: GENERATE CLIENT_EXT.GO
|
||||
# ============================================================================
|
||||
|
||||
def parse_oas_client_methods(client_file: str) -> dict:
|
||||
"""Parse method signatures from oas_client_gen.go"""
|
||||
with open(client_file, 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
methods = {}
|
||||
pattern = r'func \(c \*Client\) (\w+)\((ctx context\.Context(?:,\s*[^)]+)?)\)\s*\(([^)]+)\)'
|
||||
|
||||
for match in re.finditer(pattern, content, re.MULTILINE):
|
||||
method_name = match.group(1)
|
||||
if method_name in ['requestURL'] or method_name.startswith('send'):
|
||||
continue
|
||||
|
||||
full_params = match.group(2)
|
||||
returns = match.group(3)
|
||||
|
||||
# Parse params (skip ctx and variadic options)
|
||||
params_list = []
|
||||
has_options = False
|
||||
if ', ' in full_params:
|
||||
params_str = full_params.split(', ', 1)[1]
|
||||
# Detect variadic ...RequestOption
|
||||
if '...RequestOption' in params_str:
|
||||
has_options = True
|
||||
# Remove variadic param before parsing regular params
|
||||
params_str = re.sub(r',?\s*options\s+\.\.\.RequestOption', '', params_str).strip()
|
||||
for param in re.findall(r'(\w+)\s+([\*\w\.]+)', params_str):
|
||||
params_list.append((param[0], param[1]))
|
||||
|
||||
returns_list = [r.strip() for r in returns.split(',')]
|
||||
|
||||
methods[method_name] = {
|
||||
'params': params_list,
|
||||
'returns': returns_list,
|
||||
'has_options': has_options,
|
||||
}
|
||||
|
||||
return methods
|
||||
|
||||
|
||||
def parse_params_structs(params_file: str) -> dict:
|
||||
"""Parse Params struct fields from oas_parameters_gen.go"""
|
||||
with open(params_file, 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
params_structs = {}
|
||||
|
||||
# Match struct definitions with their fields
|
||||
# Pattern: type XXXParams struct {\n\tField Type\n}
|
||||
pattern = r'type (\w+Params) struct \{([^}]*)\}'
|
||||
|
||||
for match in re.finditer(pattern, content, re.DOTALL):
|
||||
struct_name = match.group(1)
|
||||
fields_block = match.group(2)
|
||||
|
||||
fields = []
|
||||
# Parse fields: Name Type or Name Type `json:"..."`
|
||||
for line in fields_block.strip().split('\n'):
|
||||
line = line.strip()
|
||||
if not line or line.startswith('//'):
|
||||
continue
|
||||
# Match field: UUID string or Size OptFloat64
|
||||
field_match = re.match(r'^(\w+)\s+([\w\.\*\[\]]+)', line)
|
||||
if field_match:
|
||||
field_name = field_match.group(1)
|
||||
field_type = field_match.group(2)
|
||||
fields.append((field_name, field_type))
|
||||
|
||||
params_structs[struct_name] = fields
|
||||
|
||||
return params_structs
|
||||
|
||||
|
||||
def simplify_param_type(param_type: str) -> str:
|
||||
"""Convert ogen types to simpler Go types for method signatures"""
|
||||
# OptString -> string, OptFloat64 -> float64, etc.
|
||||
type_map = {
|
||||
'OptString': 'string',
|
||||
'OptInt': 'int',
|
||||
'OptFloat64': 'float64',
|
||||
'OptBool': 'bool',
|
||||
}
|
||||
return type_map.get(param_type, param_type)
|
||||
|
||||
|
||||
# Go reserved keywords that cannot be used as identifiers
|
||||
GO_KEYWORDS = {
|
||||
'break', 'case', 'chan', 'const', 'continue', 'default', 'defer', 'else',
|
||||
'fallthrough', 'for', 'func', 'go', 'goto', 'if', 'import', 'interface',
|
||||
'map', 'package', 'range', 'return', 'select', 'struct', 'switch', 'type',
|
||||
'var',
|
||||
}
|
||||
|
||||
|
||||
def safe_param_name(name: str) -> str:
|
||||
"""Convert a field name to a safe Go parameter name, avoiding reserved keywords."""
|
||||
lower = name.lower()
|
||||
if lower in GO_KEYWORDS:
|
||||
return lower + 'Val'
|
||||
return lower
|
||||
|
||||
|
||||
def _to_pascal(s: str) -> str:
|
||||
"""Convert first letter to uppercase, preserving camelCase."""
|
||||
if not s:
|
||||
return s
|
||||
return s[0].upper() + s[1:]
|
||||
|
||||
|
||||
def parse_operations(spec_file: str) -> dict:
|
||||
"""Parse operations from OpenAPI spec"""
|
||||
with open(spec_file, 'r') as f:
|
||||
spec = json.load(f)
|
||||
|
||||
operations_by_controller = {}
|
||||
|
||||
for path, path_item in spec.get('paths', {}).items():
|
||||
for http_method, op_spec in path_item.items():
|
||||
if http_method not in ['get', 'post', 'put', 'patch', 'delete']:
|
||||
continue
|
||||
|
||||
op_id = op_spec.get('operationId')
|
||||
if not op_id or '_' not in op_id:
|
||||
continue
|
||||
|
||||
parts = op_id.split('_', 1)
|
||||
controller_full = parts[0]
|
||||
method_snake = parts[1]
|
||||
|
||||
controller = controller_full.replace('Controller', '')
|
||||
|
||||
method_parts = method_snake.split('_')
|
||||
method_pascal = ''.join(_to_pascal(p) for p in method_parts)
|
||||
|
||||
go_method = controller_full + method_pascal
|
||||
|
||||
if controller not in operations_by_controller:
|
||||
operations_by_controller[controller] = []
|
||||
|
||||
operations_by_controller[controller].append({
|
||||
'operationId': op_id,
|
||||
'goMethod': go_method,
|
||||
'displayMethod': method_pascal
|
||||
})
|
||||
|
||||
return operations_by_controller
|
||||
|
||||
|
||||
def generate_client_ext(spec_file: str, client_file: str, output_file: str) -> Tuple[int, int]:
|
||||
"""Generate client_ext.go wrapper with simplified method signatures"""
|
||||
print_info("Parsing oas_client_gen.go...")
|
||||
methods = parse_oas_client_methods(client_file)
|
||||
print_success(f"Found {len(methods)} client methods")
|
||||
|
||||
# Parse params structs for simplification
|
||||
params_file = client_file.replace('oas_client_gen.go', 'oas_parameters_gen.go')
|
||||
print_info("Parsing oas_parameters_gen.go...")
|
||||
params_structs = parse_params_structs(params_file)
|
||||
print_success(f"Found {len(params_structs)} param structs")
|
||||
|
||||
print_info("Parsing operations from spec...")
|
||||
operations_by_controller = parse_operations(spec_file)
|
||||
total_ops = sum(len(ops) for ops in operations_by_controller.values())
|
||||
print_success(f"Found {total_ops} operations in {len(operations_by_controller)} controllers")
|
||||
|
||||
def to_camel(s):
|
||||
return s[0].lower() + s[1:] if s else s
|
||||
|
||||
def can_simplify_params(params_type: str) -> tuple:
|
||||
"""
|
||||
Check if Params struct can be simplified to individual arguments.
|
||||
Returns (can_simplify, [(field_name, field_type, simple_type), ...])
|
||||
"""
|
||||
struct_name = params_type.lstrip('*')
|
||||
if struct_name not in params_structs:
|
||||
return False, []
|
||||
|
||||
fields = params_structs[struct_name]
|
||||
if not fields:
|
||||
return False, []
|
||||
|
||||
# Only simplify if all fields are simple types
|
||||
simple_types = {'string', 'int', 'int64', 'float64', 'bool',
|
||||
'OptString', 'OptInt', 'OptInt64', 'OptFloat64', 'OptBool'}
|
||||
|
||||
simplified = []
|
||||
for field_name, field_type in fields:
|
||||
if field_type in simple_types or field_type.startswith('Opt'):
|
||||
simple = simplify_param_type(field_type)
|
||||
simplified.append((field_name, field_type, simple))
|
||||
else:
|
||||
# Complex type, don't simplify
|
||||
return False, []
|
||||
|
||||
return True, simplified
|
||||
|
||||
# Generate code
|
||||
code = '''// Code generated by pipeline.py. DO NOT EDIT manually.
|
||||
|
||||
package api
|
||||
|
||||
import "context"
|
||||
|
||||
// ClientExt wraps the base Client with organized sub-client access.
|
||||
// Use controller methods (e.g., client.Users().GetByUuid()) to call API operations.
|
||||
type ClientExt struct {
|
||||
\tclient *Client
|
||||
'''
|
||||
|
||||
for controller in sorted(operations_by_controller.keys()):
|
||||
field_name = to_camel(controller)
|
||||
code += f'\t{field_name} *{controller}Client\n'
|
||||
|
||||
code += '''}
|
||||
|
||||
// NewClientExt creates a new ClientExt wrapper.
|
||||
func NewClientExt(client *Client) *ClientExt {
|
||||
\treturn &ClientExt{
|
||||
\t\tclient: client,
|
||||
'''
|
||||
|
||||
for controller in sorted(operations_by_controller.keys()):
|
||||
field_name = to_camel(controller)
|
||||
code += f'\t\t{field_name}: New{controller}Client(client),\n'
|
||||
|
||||
code += '''\t}
|
||||
}
|
||||
|
||||
// Client returns the underlying ogen Client.
|
||||
func (ce *ClientExt) Client() *Client {
|
||||
\treturn ce.client
|
||||
}
|
||||
|
||||
'''
|
||||
|
||||
for controller in sorted(operations_by_controller.keys()):
|
||||
field_name = to_camel(controller)
|
||||
code += f'''// {controller} returns the {controller}Client.
|
||||
func (ce *ClientExt) {controller}() *{controller}Client {{
|
||||
\treturn ce.{field_name}
|
||||
}}
|
||||
|
||||
'''
|
||||
|
||||
matched_methods = 0
|
||||
|
||||
for controller in sorted(operations_by_controller.keys()):
|
||||
code += f'''// {controller}Client provides {controller} operations.
|
||||
type {controller}Client struct {{
|
||||
\tclient *Client
|
||||
}}
|
||||
|
||||
// New{controller}Client creates a new {controller}Client.
|
||||
func New{controller}Client(client *Client) *{controller}Client {{
|
||||
\treturn &{controller}Client{{client: client}}
|
||||
}}
|
||||
|
||||
'''
|
||||
|
||||
for op in sorted(operations_by_controller[controller], key=lambda x: x['goMethod']):
|
||||
go_method = op['goMethod']
|
||||
display_method = op['displayMethod']
|
||||
op_id = op['operationId']
|
||||
|
||||
if go_method not in methods:
|
||||
continue
|
||||
|
||||
matched_methods += 1
|
||||
method_info = methods[go_method]
|
||||
params = method_info['params']
|
||||
returns = method_info['returns']
|
||||
has_options = method_info.get('has_options', False)
|
||||
|
||||
# options suffix for signature and call
|
||||
opts_sig = ', options ...RequestOption' if has_options else ''
|
||||
opts_call = ', options...' if has_options else ''
|
||||
|
||||
# Check if we can simplify Params struct to individual args
|
||||
simplified_params = None
|
||||
params_index = None
|
||||
for i, (pname, ptype) in enumerate(params):
|
||||
if ptype.endswith('Params'):
|
||||
can_simplify, simplified = can_simplify_params(ptype)
|
||||
if can_simplify:
|
||||
simplified_params = simplified
|
||||
params_index = i
|
||||
break
|
||||
|
||||
if returns:
|
||||
ret_type = ', '.join(returns)
|
||||
if len(returns) > 1:
|
||||
ret_type = f'({ret_type})'
|
||||
else:
|
||||
ret_type = ''
|
||||
|
||||
# Generate method with simplified params or original
|
||||
if simplified_params and params_index is not None:
|
||||
params_type = params[params_index][1]
|
||||
|
||||
sig_parts = []
|
||||
for i, (pname, ptype) in enumerate(params):
|
||||
if i == params_index:
|
||||
for field_name, field_type, simple_type in simplified_params:
|
||||
sig_parts.append(f'{safe_param_name(field_name)} {simple_type}')
|
||||
else:
|
||||
sig_parts.append(f'{pname} {ptype}')
|
||||
|
||||
simple_args = ', '.join(sig_parts)
|
||||
|
||||
params_init = f'{params_type}{{\n'
|
||||
for field_name, field_type, simple_type in simplified_params:
|
||||
arg_name = safe_param_name(field_name)
|
||||
if field_type.startswith('Opt'):
|
||||
params_init += f'\t\t{field_name}: NewOpt{simple_type.title()}({arg_name}),\n'
|
||||
else:
|
||||
params_init += f'\t\t{field_name}: {arg_name},\n'
|
||||
params_init += '\t}'
|
||||
|
||||
call_args = []
|
||||
for i, (pname, ptype) in enumerate(params):
|
||||
if i == params_index:
|
||||
call_args.append(params_init)
|
||||
else:
|
||||
call_args.append(pname)
|
||||
|
||||
code += f'''// {display_method} calls {op_id}.
|
||||
func (sc *{controller}Client) {display_method}(ctx context.Context, {simple_args}{opts_sig}) {ret_type} {{
|
||||
\treturn sc.client.{go_method}(ctx, {', '.join(call_args)}{opts_call})
|
||||
}}
|
||||
|
||||
'''
|
||||
else:
|
||||
# Original params
|
||||
if params:
|
||||
params_sig = ', '.join([f'{p[0]} {p[1]}' for p in params])
|
||||
params_call = ', '.join([p[0] for p in params])
|
||||
else:
|
||||
params_sig = ''
|
||||
params_call = ''
|
||||
|
||||
code += f'''// {display_method} calls {op_id}.
|
||||
func (sc *{controller}Client) {display_method}(ctx context.Context'''
|
||||
|
||||
if params_sig:
|
||||
code += f', {params_sig}'
|
||||
|
||||
code += opts_sig + ')'
|
||||
|
||||
if ret_type:
|
||||
code += f' {ret_type}'
|
||||
|
||||
code += ' {\n'
|
||||
|
||||
if returns:
|
||||
code += f'\treturn sc.client.{go_method}(ctx'
|
||||
else:
|
||||
code += f'\tsc.client.{go_method}(ctx'
|
||||
|
||||
if params_call:
|
||||
code += f', {params_call}'
|
||||
|
||||
code += opts_call + ')\n}\n\n'
|
||||
|
||||
print_info(f"Writing {output_file}...")
|
||||
with open(output_file, 'w') as f:
|
||||
f.write(code)
|
||||
|
||||
print_success(f"Generated {matched_methods}/{total_ops} methods")
|
||||
|
||||
return len(operations_by_controller), matched_methods
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# MAIN PIPELINE
|
||||
# ============================================================================
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
print_error("Usage: python3 pipeline.py <input_spec.json>")
|
||||
sys.exit(1)
|
||||
|
||||
input_spec = sys.argv[1]
|
||||
|
||||
if not Path(input_spec).exists():
|
||||
print_error(f"File not found: {input_spec}")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"{Colors.BOLD}{Colors.HEADER}")
|
||||
print("="*70)
|
||||
print(" API PROCESSING PIPELINE")
|
||||
print("="*70)
|
||||
print(f"{Colors.END}")
|
||||
print(f"Input: {input_spec}")
|
||||
|
||||
# File paths - now we only need one output file since smart_consolidate does both steps
|
||||
final_file = input_spec.replace('.json', '-final.json')
|
||||
client_gen_file = 'api/oas_client_gen.go'
|
||||
client_ext_file = 'api/client_ext.go'
|
||||
|
||||
try:
|
||||
# Step 1: Smart consolidate (combines old Steps 1 & 2)
|
||||
print_step(1, 3, "SMART CONSOLIDATE SCHEMAS")
|
||||
orig_count, new_count, stats = smart_consolidate_schemas(input_spec, final_file)
|
||||
|
||||
# Step 1.5: Post-process the consolidated spec (in-memory)
|
||||
print_info("Post-processing consolidated spec...")
|
||||
with open(final_file, 'r') as f:
|
||||
final_spec = json.load(f)
|
||||
|
||||
patched_count = patch_subscription_text_responses(final_spec)
|
||||
if patched_count > 0:
|
||||
print_success(f"Patched {patched_count} subscription endpoints with text/plain response")
|
||||
|
||||
renamed_count = shorten_operation_ids(final_spec)
|
||||
if renamed_count > 0:
|
||||
print_success(f"Shortened {renamed_count} operationIds (removed 'Controller')")
|
||||
|
||||
dto_count = strip_dto_suffix(final_spec)
|
||||
if dto_count > 0:
|
||||
print_success(f"Stripped 'Dto' suffix from {dto_count} schema names")
|
||||
|
||||
int_count = fix_number_query_params(final_spec)
|
||||
if int_count > 0:
|
||||
print_success(f"Fixed {int_count} query parameters: number → integer")
|
||||
|
||||
with open(final_file, 'w') as f:
|
||||
json.dump(final_spec, f, indent=2, ensure_ascii=False)
|
||||
|
||||
# Step 2: Generate with ogen
|
||||
print_step(2, 3, "GENERATE GO CLIENT WITH OGEN")
|
||||
if not generate_ogen_client(final_file):
|
||||
print_error("Failed to generate Go client")
|
||||
sys.exit(1)
|
||||
|
||||
# Step 3: Generate client_ext
|
||||
print_step(3, 3, "GENERATE CLIENT_EXT.GO WRAPPER")
|
||||
ctrl_count, method_count = generate_client_ext(final_file, client_gen_file, client_ext_file)
|
||||
|
||||
# Summary
|
||||
print(f"\n{Colors.BOLD}{Colors.GREEN}")
|
||||
print("="*70)
|
||||
print(" PIPELINE COMPLETED SUCCESSFULLY")
|
||||
print("="*70)
|
||||
print(f"{Colors.END}")
|
||||
print(f"\n{Colors.BOLD}Results:{Colors.END}")
|
||||
print(f" • Schemas: {orig_count} → {new_count} (-{orig_count - new_count}, -{(orig_count-new_count)*100//orig_count}%)")
|
||||
print(f" • Groups: {stats.get('duplicate_groups', 0)} consolidated")
|
||||
print(f" • Controllers: {ctrl_count}")
|
||||
print(f" • Methods: {method_count}")
|
||||
print(f"\n{Colors.BOLD}Generated files:{Colors.END}")
|
||||
print(f" • {final_file}")
|
||||
print(f" • {client_gen_file}")
|
||||
print(f" • {client_ext_file}")
|
||||
print()
|
||||
|
||||
except Exception as e:
|
||||
print_error(f"Pipeline failed: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+243
@@ -0,0 +1,243 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Rename consolidated schemas to more common naming conventions.
|
||||
|
||||
Changes patterns like:
|
||||
- CreateUserResponseDto → UserResponse
|
||||
- DeleteResponseDto → DeleteResponse
|
||||
- EventResponseDto → EventResponse
|
||||
- BulkActionResponseDto → BulkActionResponse
|
||||
- BulkUuidsRequestDto → BulkUuidsRequest
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def create_rename_map() -> dict:
|
||||
"""Create mapping from old names to new common names."""
|
||||
return {
|
||||
# User responses (9 schemas)
|
||||
'CreateUserResponseDto': 'UserResponse',
|
||||
'DisableUserResponseDto': 'UserResponse',
|
||||
'EnableUserResponseDto': 'UserResponse',
|
||||
'GetUserByShortUuidResponseDto': 'UserResponse',
|
||||
'GetUserByUsernameResponseDto': 'UserResponse',
|
||||
'GetUserByUuidResponseDto': 'UserResponse',
|
||||
'ResetUserTrafficResponseDto': 'UserResponse',
|
||||
'RevokeUserSubscriptionResponseDto': 'UserResponse',
|
||||
'UpdateUserResponseDto': 'UserResponse',
|
||||
|
||||
# Delete operations (8 schemas)
|
||||
'DeleteConfigProfileResponseDto': 'DeleteResponse',
|
||||
'DeleteExternalSquadResponseDto': 'DeleteResponse',
|
||||
'DeleteHostResponseDto': 'DeleteResponse',
|
||||
'DeleteInfraProviderByUuidResponseDto': 'DeleteResponse',
|
||||
'DeleteInternalSquadResponseDto': 'DeleteResponse',
|
||||
'DeleteNodeResponseDto': 'DeleteResponse',
|
||||
'DeleteSubscriptionTemplateResponseDto': 'DeleteResponse',
|
||||
'DeleteUserResponseDto': 'DeleteResponse',
|
||||
'DeletePasskeyResponseDto': 'DeleteResponse',
|
||||
|
||||
# Event operations (8 schemas)
|
||||
'AddUsersToExternalSquadResponseDto': 'EventResponse',
|
||||
'AddUsersToInternalSquadResponseDto': 'EventResponse',
|
||||
'BulkAllResetTrafficUsersResponseDto': 'EventResponse',
|
||||
'BulkAllUpdateUsersResponseDto': 'EventResponse',
|
||||
'RemoveUsersFromExternalSquadResponseDto': 'EventResponse',
|
||||
'RemoveUsersFromInternalSquadResponseDto': 'EventResponse',
|
||||
'RestartAllNodesResponseDto': 'EventResponse',
|
||||
'RestartNodeResponseDto': 'EventResponse',
|
||||
|
||||
# Bulk responses (6 schemas)
|
||||
'BulkDeleteUsersByStatusResponseDto': 'BulkActionResponse',
|
||||
'BulkDeleteUsersResponseDto': 'BulkActionResponse',
|
||||
'BulkResetTrafficUsersResponseDto': 'BulkActionResponse',
|
||||
'BulkRevokeUsersSubscriptionResponseDto': 'BulkActionResponse',
|
||||
'BulkUpdateUsersResponseDto': 'BulkActionResponse',
|
||||
'BulkUpdateUsersSquadsResponseDto': 'BulkActionResponse',
|
||||
|
||||
# Bulk requests (6 schemas)
|
||||
'BulkDeleteHostsRequestDto': 'BulkUuidsRequest',
|
||||
'BulkDisableHostsRequestDto': 'BulkUuidsRequest',
|
||||
'BulkEnableHostsRequestDto': 'BulkUuidsRequest',
|
||||
'BulkResetTrafficUsersRequestDto': 'BulkUuidsRequest',
|
||||
'BulkRevokeUsersSubscriptionRequestDto': 'BulkUuidsRequest',
|
||||
'BulkUuidsRequestDto': 'BulkUuidsRequest',
|
||||
|
||||
# Hosts (6 schemas)
|
||||
'BulkDeleteHostsResponseDto': 'HostListResponse',
|
||||
'BulkDisableHostsResponseDto': 'HostListResponse',
|
||||
'BulkEnableHostsResponseDto': 'HostListResponse',
|
||||
'GetAllHostsResponseDto': 'HostListResponse',
|
||||
'SetInboundToManyHostsResponseDto': 'HostListResponse',
|
||||
'SetPortToManyHostsResponseDto': 'HostListResponse',
|
||||
|
||||
# Auth tokens (5 schemas)
|
||||
'LoginResponseDto': 'TokenResponse',
|
||||
'OAuth2CallbackResponseDto': 'TokenResponse',
|
||||
'RegisterResponseDto': 'TokenResponse',
|
||||
'TelegramCallbackResponseDto': 'TokenResponse',
|
||||
'VerifyPasskeyAuthenticationResponseDto': 'TokenResponse',
|
||||
|
||||
# Node responses (5 schemas)
|
||||
'CreateNodeResponseDto': 'NodeResponse',
|
||||
'DisableNodeResponseDto': 'NodeResponse',
|
||||
'EnableNodeResponseDto': 'NodeResponse',
|
||||
'GetOneNodeResponseDto': 'NodeResponse',
|
||||
'UpdateNodeResponseDto': 'NodeResponse',
|
||||
|
||||
# Passkey/Auth
|
||||
'GetPasskeyRegistrationOptionsResponseDto': 'PasskeyOptionsResponse',
|
||||
'GetPasskeyAuthenticationOptionsResponseDto': 'PasskeyOptionsResponse',
|
||||
'VerifyPasskeyAuthenticationRequestDto': 'PasskeyOptionsResponse',
|
||||
'VerifyPasskeyRegistrationRequestDto': 'PasskeyOptionsResponse',
|
||||
|
||||
# Subscriptions (4 schemas)
|
||||
'GetSubscriptionByShortUuidProtectedResponseDto': 'SubscriptionResponse',
|
||||
'GetSubscriptionByUsernameResponseDto': 'SubscriptionResponse',
|
||||
'GetSubscriptionByUuidResponseDto': 'SubscriptionResponse',
|
||||
'GetSubscriptionInfoResponseDto': 'SubscriptionResponse',
|
||||
|
||||
# Snippets (4 schemas)
|
||||
'CreateSnippetResponseDto': 'SnippetsResponse',
|
||||
'DeleteSnippetResponseDto': 'SnippetsResponse',
|
||||
'GetSnippetsResponseDto': 'SnippetsResponse',
|
||||
'UpdateSnippetResponseDto': 'SnippetsResponse',
|
||||
|
||||
# HWID Devices (4 schemas)
|
||||
'CreateUserHwidDeviceResponseDto': 'HwidDevicesResponse',
|
||||
'DeleteAllUserHwidDevicesResponseDto': 'HwidDevicesResponse',
|
||||
'DeleteUserHwidDeviceResponseDto': 'HwidDevicesResponse',
|
||||
'GetUserHwidDevicesResponseDto': 'HwidDevicesResponse',
|
||||
|
||||
# Billing Nodes (4 schemas)
|
||||
'CreateInfraBillingNodeResponseDto': 'BillingNodesResponse',
|
||||
'DeleteInfraBillingNodeByUuidResponseDto': 'BillingNodesResponse',
|
||||
'GetInfraBillingNodesResponseDto': 'BillingNodesResponse',
|
||||
'UpdateInfraBillingNodeResponseDto': 'BillingNodesResponse',
|
||||
|
||||
# Other mappings for remaining schemas
|
||||
'GetUserByEmailResponseDto': 'UsersResponse',
|
||||
'GetUserByTagResponseDto': 'UsersResponse',
|
||||
'GetUserByTelegramIdResponseDto': 'UsersResponse',
|
||||
|
||||
'CreateSubscriptionTemplateResponseDto': 'TemplateResponse',
|
||||
'GetTemplateResponseDto': 'TemplateResponse',
|
||||
'UpdateTemplateResponseDto': 'TemplateResponse',
|
||||
|
||||
'CreateConfigProfileResponseDto': 'ConfigProfileResponse',
|
||||
'GetConfigProfileByUuidResponseDto': 'ConfigProfileResponse',
|
||||
'UpdateConfigProfileResponseDto': 'ConfigProfileResponse',
|
||||
|
||||
'CreateInternalSquadResponseDto': 'InternalSquadResponse',
|
||||
'GetInternalSquadByUuidResponseDto': 'InternalSquadResponse',
|
||||
'UpdateInternalSquadResponseDto': 'InternalSquadResponse',
|
||||
|
||||
'CreateExternalSquadResponseDto': 'ExternalSquadResponse',
|
||||
'GetExternalSquadByUuidResponseDto': 'ExternalSquadResponse',
|
||||
'UpdateExternalSquadResponseDto': 'ExternalSquadResponse',
|
||||
|
||||
'CreateHostResponseDto': 'HostResponse',
|
||||
'GetOneHostResponseDto': 'HostResponse',
|
||||
'UpdateHostResponseDto': 'HostResponse',
|
||||
|
||||
'CreateInfraProviderResponseDto': 'InfraProviderResponse',
|
||||
'GetInfraProviderByUuidResponseDto': 'InfraProviderResponse',
|
||||
'UpdateInfraProviderResponseDto': 'InfraProviderResponse',
|
||||
|
||||
'CreateInfraBillingHistoryRecordResponseDto': 'BillingHistoryResponse',
|
||||
'DeleteInfraBillingHistoryRecordByUuidResponseDto': 'BillingHistoryResponse',
|
||||
'GetInfraBillingHistoryRecordsResponseDto': 'BillingHistoryResponse',
|
||||
|
||||
'GetRemnawaveSettingsResponseDto': 'SettingsResponse',
|
||||
'UpdateRemnawaveSettingsResponseDto': 'SettingsResponse',
|
||||
|
||||
'GetAllPasskeysResponseDto': 'PasskeysResponse',
|
||||
|
||||
'GetAllTagsResponseDto': 'TagsResponse',
|
||||
'GetAllHostTagsResponseDto': 'TagsResponse',
|
||||
|
||||
'GetAllInboundsResponseDto': 'InboundsResponse',
|
||||
'GetInboundsByProfileUuidResponseDto': 'InboundsResponse',
|
||||
|
||||
'CreateSnippetRequestDto': 'SnippetRequest',
|
||||
'UpdateSnippetRequestDto': 'SnippetRequest',
|
||||
|
||||
'GetAllNodesResponseDto': 'NodesResponse',
|
||||
'ReorderNodeResponseDto': 'NodesResponse',
|
||||
|
||||
'GetSubscriptionSettingsResponseDto': 'SubscriptionSettingsResponse',
|
||||
'UpdateSubscriptionSettingsResponseDto': 'SubscriptionSettingsResponse',
|
||||
}
|
||||
|
||||
|
||||
def rename_schemas_in_spec(spec: dict, rename_map: dict) -> dict:
|
||||
"""Rename all schemas in the OpenAPI spec."""
|
||||
schemas = spec.get('components', {}).get('schemas', {})
|
||||
|
||||
new_schemas = {}
|
||||
for old_name, schema_def in schemas.items():
|
||||
new_name = rename_map.get(old_name, old_name)
|
||||
new_schemas[new_name] = schema_def
|
||||
|
||||
spec['components']['schemas'] = new_schemas
|
||||
return spec
|
||||
|
||||
|
||||
def update_schema_references(spec: dict, rename_map: dict) -> dict:
|
||||
"""Update all $ref references to use new schema names."""
|
||||
|
||||
def replace_refs(obj):
|
||||
if isinstance(obj, dict):
|
||||
for key, value in obj.items():
|
||||
if key == '$ref' and isinstance(value, str):
|
||||
if value.startswith('#/components/schemas/'):
|
||||
old_name = value.replace('#/components/schemas/', '')
|
||||
new_name = rename_map.get(old_name, old_name)
|
||||
obj[key] = f'#/components/schemas/{new_name}'
|
||||
else:
|
||||
replace_refs(value)
|
||||
elif isinstance(obj, list):
|
||||
for item in obj:
|
||||
replace_refs(item)
|
||||
|
||||
replace_refs(spec)
|
||||
return spec
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python3 rename_schemas.py <input_file> [output_file]")
|
||||
print("Example: python3 rename_schemas.py api-2-2-2-consolidated.json api-2-2-2-renamed.json")
|
||||
sys.exit(1)
|
||||
|
||||
input_file = sys.argv[1]
|
||||
output_file = sys.argv[2] if len(sys.argv) > 2 else input_file.replace('.json', '-renamed.json')
|
||||
|
||||
print(f"📂 Loading {input_file}...")
|
||||
with open(input_file, 'r') as f:
|
||||
spec = json.load(f)
|
||||
|
||||
rename_map = create_rename_map()
|
||||
|
||||
print(f"🔄 Renaming {len(rename_map)} schemas to common names...")
|
||||
spec = rename_schemas_in_spec(spec, rename_map)
|
||||
|
||||
print(f"🔗 Updating all schema references...")
|
||||
spec = update_schema_references(spec, rename_map)
|
||||
|
||||
print(f"💾 Saving to {output_file}...")
|
||||
with open(output_file, 'w') as f:
|
||||
json.dump(spec, f, indent=2, ensure_ascii=False)
|
||||
|
||||
print(f"✅ Done! Renamed schemas saved to {output_file}")
|
||||
print(f"\nSchema name mappings applied:")
|
||||
for old, new in sorted(rename_map.items()):
|
||||
if old != new:
|
||||
print(f" {old} → {new}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+1268
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user