Coverage for transformer_lens/tools/model_registry/validate.py: 56%

1"""JSON schema validation for the model registry output files. 

2 

3This module provides functions to validate that the JSON output files in the data/ 

4directory conform to the expected schemas defined by the dataclasses in schemas.py 

5and verification.py. 

6""" 

7 

8import json 

9import logging 

10from dataclasses import dataclass 

11from datetime import date, datetime 

12from pathlib import Path 

13from typing import Any 

14 

15logger = logging.getLogger(__name__) 

16 

17 

18@dataclass 

19class ValidationError: 

20 """Represents a validation error in a JSON file. 

21 

22 Attributes: 

23 path: JSON path where the error occurred (e.g., "models[0].architecture_id") 

24 message: Description of the validation error 

25 value: The actual value that caused the error (if applicable) 

26 """ 

27 

28 path: str 

29 message: str 

30 value: Any = None 

31 

32 def __str__(self) -> str: 

33 """Return a human-readable error message.""" 

34 if self.value is not None: 

35 return f"{self.path}: {self.message} (got: {self.value!r})" 

36 return f"{self.path}: {self.message}" 

37 

38 

39@dataclass 

40class ValidationResult: 

41 """Result of validating a JSON file against its schema. 

42 

43 Attributes: 

44 valid: Whether the file passed validation 

45 errors: List of validation errors (empty if valid) 

46 schema_type: The schema type that was validated against 

47 """ 

48 

49 valid: bool 

50 errors: list[ValidationError] 

51 schema_type: str 

52 

53 @property 

54 def error_count(self) -> int: 

55 """Return the number of validation errors.""" 

56 return len(self.errors) 

57 

58 

59def _validate_string( 

60 value: Any, path: str, required: bool = True, min_length: int = 0 

61) -> list[ValidationError]: 

62 """Validate that a value is a string. 

63 

64 Args: 

65 value: The value to validate 

66 path: JSON path for error reporting 

67 required: Whether the field is required (None not allowed) 

68 min_length: Minimum string length (only checked if value is not None) 

69 

70 Returns: 

71 List of validation errors (empty if valid) 

72 """ 

73 errors = [] 

74 if value is None: 74 ↛ 75line 74 didn't jump to line 75 because the condition on line 74 was never true

75 if required: 

76 errors.append(ValidationError(path, "required field is missing or null")) 

77 elif not isinstance(value, str): 77 ↛ 78line 77 didn't jump to line 78 because the condition on line 77 was never true

78 errors.append(ValidationError(path, f"expected string, got {type(value).__name__}", value)) 

79 elif min_length > 0 and len(value) < min_length: 

80 errors.append( 

81 ValidationError(path, f"string must be at least {min_length} characters", value) 

82 ) 

83 return errors 

84 

85 

86def _validate_int( 

87 value: Any, path: str, required: bool = True, min_value: int | None = None 

88) -> list[ValidationError]: 

89 """Validate that a value is an integer. 

90 

91 Args: 

92 value: The value to validate 

93 path: JSON path for error reporting 

94 required: Whether the field is required (None not allowed) 

95 min_value: Minimum allowed value (only checked if value is not None) 

96 

97 Returns: 

98 List of validation errors (empty if valid) 

99 """ 

100 errors = [] 

101 if value is None: 

102 if required: 102 ↛ 108line 102 didn't jump to line 108 because the condition on line 102 was always true

103 errors.append(ValidationError(path, "required field is missing or null")) 

104 elif not isinstance(value, int) or isinstance(value, bool): 104 ↛ 105line 104 didn't jump to line 105 because the condition on line 104 was never true

105 errors.append(ValidationError(path, f"expected integer, got {type(value).__name__}", value)) 

106 elif min_value is not None and value < min_value: 106 ↛ 107line 106 didn't jump to line 107 because the condition on line 106 was never true

107 errors.append(ValidationError(path, f"value must be >= {min_value}", value)) 

108 return errors 

109 

110 

111def _validate_bool(value: Any, path: str, required: bool = True) -> list[ValidationError]: 

112 """Validate that a value is a boolean. 

113 

114 Args: 

115 value: The value to validate 

116 path: JSON path for error reporting 

117 required: Whether the field is required (None not allowed) 

118 

119 Returns: 

120 List of validation errors (empty if valid) 

121 """ 

122 errors = [] 

123 if value is None: 123 ↛ 124line 123 didn't jump to line 124 because the condition on line 123 was never true

124 if required: 

125 errors.append(ValidationError(path, "required field is missing or null")) 

126 elif not isinstance(value, bool): 126 ↛ 127line 126 didn't jump to line 127 because the condition on line 126 was never true

127 errors.append(ValidationError(path, f"expected boolean, got {type(value).__name__}", value)) 

128 return errors 

129 

130 

131def _validate_date_string(value: Any, path: str, required: bool = True) -> list[ValidationError]: 

132 """Validate that a value is a valid ISO date string. 

133 

134 Args: 

135 value: The value to validate 

136 path: JSON path for error reporting 

137 required: Whether the field is required (None not allowed) 

138 

139 Returns: 

140 List of validation errors (empty if valid) 

141 """ 

142 errors = [] 

143 if value is None: 

144 if required: 144 ↛ 155line 144 didn't jump to line 155 because the condition on line 144 was always true

145 errors.append(ValidationError(path, "required field is missing or null")) 

146 elif not isinstance(value, str): 146 ↛ 147line 146 didn't jump to line 147 because the condition on line 146 was never true

147 errors.append( 

148 ValidationError(path, f"expected date string, got {type(value).__name__}", value) 

149 ) 

150 else: 

151 try: 

152 date.fromisoformat(value) 

153 except ValueError: 

154 errors.append(ValidationError(path, "invalid ISO date format", value)) 

155 return errors 

156 

157 

158def _validate_datetime_string( 

159 value: Any, path: str, required: bool = True 

160) -> list[ValidationError]: 

161 """Validate that a value is a valid ISO datetime string. 

162 

163 Args: 

164 value: The value to validate 

165 path: JSON path for error reporting 

166 required: Whether the field is required (None not allowed) 

167 

168 Returns: 

169 List of validation errors (empty if valid) 

170 """ 

171 errors = [] 

172 if value is None: 172 ↛ 173line 172 didn't jump to line 173 because the condition on line 172 was never true

173 if required: 

174 errors.append(ValidationError(path, "required field is missing or null")) 

175 elif not isinstance(value, str): 175 ↛ 176line 175 didn't jump to line 176 because the condition on line 175 was never true

176 errors.append( 

177 ValidationError(path, f"expected datetime string, got {type(value).__name__}", value) 

178 ) 

179 else: 

180 try: 

181 datetime.fromisoformat(value) 

182 except ValueError: 

183 errors.append(ValidationError(path, "invalid ISO datetime format", value)) 

184 return errors 

185 

186 

187def _validate_list(value: Any, path: str, required: bool = True) -> list[ValidationError]: 

188 """Validate that a value is a list. 

189 

190 Args: 

191 value: The value to validate 

192 path: JSON path for error reporting 

193 required: Whether the field is required (None not allowed) 

194 

195 Returns: 

196 List of validation errors (empty if valid) 

197 """ 

198 errors = [] 

199 if value is None: 199 ↛ 200line 199 didn't jump to line 200 because the condition on line 199 was never true

200 if required: 

201 errors.append(ValidationError(path, "required field is missing or null")) 

202 elif not isinstance(value, list): 202 ↛ 203line 202 didn't jump to line 203 because the condition on line 202 was never true

203 errors.append(ValidationError(path, f"expected list, got {type(value).__name__}", value)) 

204 return errors 

205 

206 

207def _validate_model_metadata(data: dict, path: str) -> list[ValidationError]: 

208 """Validate a ModelMetadata object. 

209 

210 Args: 

211 data: Dictionary to validate 

212 path: JSON path prefix for error reporting 

213 

214 Returns: 

215 List of validation errors (empty if valid) 

216 """ 

217 errors = [] 

218 

219 # downloads (optional, defaults to 0) 

220 if "downloads" in data: 

221 errors.extend( 

222 _validate_int(data["downloads"], f"{path}.downloads", required=False, min_value=0) 

223 ) 

224 

225 # likes (optional, defaults to 0) 

226 if "likes" in data: 

227 errors.extend(_validate_int(data["likes"], f"{path}.likes", required=False, min_value=0)) 

228 

229 # last_modified (optional datetime) 

230 if "last_modified" in data and data["last_modified"] is not None: 

231 errors.extend( 

232 _validate_datetime_string( 

233 data["last_modified"], f"{path}.last_modified", required=False 

234 ) 

235 ) 

236 

237 # tags (optional list of strings) 

238 if "tags" in data: 

239 tags = data["tags"] 

240 if tags is not None: 

241 errors.extend(_validate_list(tags, f"{path}.tags", required=False)) 

242 if isinstance(tags, list): 

243 for i, tag in enumerate(tags): 

244 errors.extend(_validate_string(tag, f"{path}.tags[{i}]")) 

245 

246 # parameter_count (optional int) 

247 if "parameter_count" in data and data["parameter_count"] is not None: 

248 errors.extend( 

249 _validate_int( 

250 data["parameter_count"], f"{path}.parameter_count", required=False, min_value=0 

251 ) 

252 ) 

253 

254 return errors 

255 

256 

257def _validate_model_entry(data: dict, path: str) -> list[ValidationError]: 

258 """Validate a ModelEntry object. 

259 

260 Args: 

261 data: Dictionary to validate 

262 path: JSON path prefix for error reporting 

263 

264 Returns: 

265 List of validation errors (empty if valid) 

266 """ 

267 errors = [] 

268 

269 if not isinstance(data, dict): 269 ↛ 270line 269 didn't jump to line 270 because the condition on line 269 was never true

270 return [ValidationError(path, f"expected object, got {type(data).__name__}", data)] 

271 

272 # architecture_id (required string) 

273 errors.extend( 

274 _validate_string(data.get("architecture_id"), f"{path}.architecture_id", min_length=1) 

275 ) 

276 

277 # model_id (required string) 

278 errors.extend(_validate_string(data.get("model_id"), f"{path}.model_id", min_length=1)) 

279 

280 # status (optional int 0-4, defaults to 0) 

281 if "status" in data: 281 ↛ 290line 281 didn't jump to line 290 because the condition on line 281 was always true

282 errors.extend(_validate_int(data["status"], f"{path}.status", required=False, min_value=0)) 

283 if isinstance(data["status"], int) and not isinstance(data["status"], bool): 283 ↛ 290line 283 didn't jump to line 290 because the condition on line 283 was always true

284 if data["status"] > 4: 

285 errors.append( 

286 ValidationError(f"{path}.status", "value must be 0-4", data["status"]) 

287 ) 

288 

289 # note (optional string) 

290 if "note" in data and data["note"] is not None: 290 ↛ 291line 290 didn't jump to line 291 because the condition on line 290 was never true

291 errors.extend(_validate_string(data["note"], f"{path}.note", min_length=1)) 

292 

293 # p4_scoring_version (optional sparse int; absent = old GPT-2 scale) 

294 if "p4_scoring_version" in data and data["p4_scoring_version"] is not None: 294 ↛ 295line 294 didn't jump to line 295 because the condition on line 294 was never true

295 version = data["p4_scoring_version"] 

296 if not isinstance(version, int) or isinstance(version, bool) or version < 2: 

297 errors.append( 

298 ValidationError(f"{path}.p4_scoring_version", "must be an int >= 2", version) 

299 ) 

300 

301 # prompt_profile (optional sparse string; must parse as a profile spec) 

302 if "prompt_profile" in data and data["prompt_profile"] is not None: 302 ↛ 303line 302 didn't jump to line 303 because the condition on line 302 was never true

303 errors.extend( 

304 _validate_string(data["prompt_profile"], f"{path}.prompt_profile", min_length=1) 

305 ) 

306 if isinstance(data["prompt_profile"], str): 

307 try: 

308 from transformer_lens.benchmarks.text_quality_profiles import ( 

309 ProfileSpec, 

310 ) 

311 

312 ProfileSpec.parse(data["prompt_profile"]) 

313 except ValueError as e: 

314 errors.append( 

315 ValidationError(f"{path}.prompt_profile", str(e), data["prompt_profile"]) 

316 ) 

317 

318 # verified_date (optional date string) 

319 if "verified_date" in data and data["verified_date"] is not None: 

320 errors.extend( 

321 _validate_date_string(data["verified_date"], f"{path}.verified_date", required=False) 

322 ) 

323 

324 # metadata (optional ModelMetadata) 

325 if "metadata" in data and data["metadata"] is not None: 325 ↛ 326line 325 didn't jump to line 326 because the condition on line 325 was never true

326 if not isinstance(data["metadata"], dict): 

327 errors.append( 

328 ValidationError( 

329 f"{path}.metadata", 

330 f"expected object, got {type(data['metadata']).__name__}", 

331 data["metadata"], 

332 ) 

333 ) 

334 else: 

335 errors.extend(_validate_model_metadata(data["metadata"], f"{path}.metadata")) 

336 

337 # phase scores (optional floats, 0-100 or None) 

338 for phase_field in ( 

339 "phase1_score", 

340 "phase2_score", 

341 "phase3_score", 

342 "phase4_score", 

343 "phase7_score", 

344 "phase8_score", 

345 "phase9_score", 

346 ): 

347 if phase_field in data and data[phase_field] is not None: 

348 val = data[phase_field] 

349 if not isinstance(val, (int, float)) or isinstance(val, bool): 349 ↛ 350line 349 didn't jump to line 350 because the condition on line 349 was never true

350 errors.append( 

351 ValidationError( 

352 f"{path}.{phase_field}", 

353 f"expected number, got {type(val).__name__}", 

354 val, 

355 ) 

356 ) 

357 

358 return errors 

359 

360 

361def _validate_architecture_gap(data: dict, path: str) -> list[ValidationError]: 

362 """Validate an ArchitectureGap object. 

363 

364 Args: 

365 data: Dictionary to validate 

366 path: JSON path prefix for error reporting 

367 

368 Returns: 

369 List of validation errors (empty if valid) 

370 """ 

371 errors = [] 

372 

373 if not isinstance(data, dict): 373 ↛ 374line 373 didn't jump to line 374 because the condition on line 373 was never true

374 return [ValidationError(path, f"expected object, got {type(data).__name__}", data)] 

375 

376 # architecture_id (required string) 

377 errors.extend( 

378 _validate_string(data.get("architecture_id"), f"{path}.architecture_id", min_length=1) 

379 ) 

380 

381 # total_models (required int >= 0) 

382 errors.extend(_validate_int(data.get("total_models"), f"{path}.total_models", min_value=0)) 

383 

384 return errors 

385 

386 

387def _validate_verification_record(data: dict, path: str) -> list[ValidationError]: 

388 """Validate a VerificationRecord object. 

389 

390 Args: 

391 data: Dictionary to validate 

392 path: JSON path prefix for error reporting 

393 

394 Returns: 

395 List of validation errors (empty if valid) 

396 """ 

397 errors = [] 

398 

399 if not isinstance(data, dict): 399 ↛ 400line 399 didn't jump to line 400 because the condition on line 399 was never true

400 return [ValidationError(path, f"expected object, got {type(data).__name__}", data)] 

401 

402 # model_id (required string) 

403 errors.extend(_validate_string(data.get("model_id"), f"{path}.model_id", min_length=1)) 

404 

405 # architecture_id (optional string, defaults to "Unknown") 

406 if "architecture_id" in data and data["architecture_id"] is not None: 406 ↛ 412line 406 didn't jump to line 412 because the condition on line 406 was always true

407 errors.extend( 

408 _validate_string(data["architecture_id"], f"{path}.architecture_id", required=False) 

409 ) 

410 

411 # verified_date (required date string) 

412 errors.extend(_validate_date_string(data.get("verified_date"), f"{path}.verified_date")) 

413 

414 # verified_by (optional string) 

415 if "verified_by" in data and data["verified_by"] is not None: 415 ↛ 419line 415 didn't jump to line 419 because the condition on line 415 was always true

416 errors.extend(_validate_string(data["verified_by"], f"{path}.verified_by", required=False)) 

417 

418 # transformerlens_version (optional string) 

419 if "transformerlens_version" in data and data["transformerlens_version"] is not None: 419 ↛ 427line 419 didn't jump to line 427 because the condition on line 419 was always true

420 errors.extend( 

421 _validate_string( 

422 data["transformerlens_version"], f"{path}.transformerlens_version", required=False 

423 ) 

424 ) 

425 

426 # notes (optional string) 

427 if "notes" in data and data["notes"] is not None: 427 ↛ 431line 427 didn't jump to line 431 because the condition on line 427 was always true

428 errors.extend(_validate_string(data["notes"], f"{path}.notes", required=False)) 

429 

430 # invalidated (optional boolean, defaults to False) 

431 if "invalidated" in data: 431 ↛ 435line 431 didn't jump to line 435 because the condition on line 431 was always true

432 errors.extend(_validate_bool(data["invalidated"], f"{path}.invalidated", required=False)) 

433 

434 # invalidation_reason (optional string) 

435 if "invalidation_reason" in data and data["invalidation_reason"] is not None: 435 ↛ 436line 435 didn't jump to line 436 because the condition on line 435 was never true

436 errors.extend( 

437 _validate_string( 

438 data["invalidation_reason"], f"{path}.invalidation_reason", required=False 

439 ) 

440 ) 

441 

442 return errors 

443 

444 

445def validate_supported_models_report(data: dict) -> ValidationResult: 

446 """Validate a SupportedModelsReport JSON object. 

447 

448 Args: 

449 data: Dictionary loaded from JSON to validate 

450 

451 Returns: 

452 ValidationResult with validation status and any errors 

453 """ 

454 errors = [] 

455 

456 if not isinstance(data, dict): 456 ↛ 457line 456 didn't jump to line 457 because the condition on line 456 was never true

457 return ValidationResult( 

458 valid=False, 

459 errors=[ 

460 ValidationError("", f"expected object at root, got {type(data).__name__}", data) 

461 ], 

462 schema_type="SupportedModelsReport", 

463 ) 

464 

465 # generated_at (required date string) 

466 errors.extend(_validate_date_string(data.get("generated_at"), "generated_at")) 

467 

468 # total_architectures (required int >= 0) 

469 errors.extend( 

470 _validate_int(data.get("total_architectures"), "total_architectures", min_value=0) 

471 ) 

472 

473 # total_models (required int >= 0) 

474 errors.extend(_validate_int(data.get("total_models"), "total_models", min_value=0)) 

475 

476 # total_verified (required int >= 0) 

477 errors.extend(_validate_int(data.get("total_verified"), "total_verified", min_value=0)) 

478 

479 # total_provisional (optional int >= 0; structural-only passes) 

480 if "total_provisional" in data: 480 ↛ 481line 480 didn't jump to line 481 because the condition on line 480 was never true

481 errors.extend( 

482 _validate_int(data.get("total_provisional"), "total_provisional", min_value=0) 

483 ) 

484 

485 # models (required list of ModelEntry) 

486 models = data.get("models") 

487 errors.extend(_validate_list(models, "models")) 

488 if isinstance(models, list): 488 ↛ 492line 488 didn't jump to line 492 because the condition on line 488 was always true

489 for i, model in enumerate(models): 

490 errors.extend(_validate_model_entry(model, f"models[{i}]")) 

491 

492 return ValidationResult( 

493 valid=len(errors) == 0, 

494 errors=errors, 

495 schema_type="SupportedModelsReport", 

496 ) 

497 

498 

499def validate_architecture_gaps_report(data: dict) -> ValidationResult: 

500 """Validate an ArchitectureGapsReport JSON object. 

501 

502 Args: 

503 data: Dictionary loaded from JSON to validate 

504 

505 Returns: 

506 ValidationResult with validation status and any errors 

507 """ 

508 errors = [] 

509 

510 if not isinstance(data, dict): 510 ↛ 511line 510 didn't jump to line 511 because the condition on line 510 was never true

511 return ValidationResult( 

512 valid=False, 

513 errors=[ 

514 ValidationError("", f"expected object at root, got {type(data).__name__}", data) 

515 ], 

516 schema_type="ArchitectureGapsReport", 

517 ) 

518 

519 # generated_at (required date string) 

520 errors.extend(_validate_date_string(data.get("generated_at"), "generated_at")) 

521 

522 # total_unsupported_architectures (required int >= 0) 

523 errors.extend( 

524 _validate_int( 

525 data.get("total_unsupported_architectures"), 

526 "total_unsupported_architectures", 

527 min_value=0, 

528 ) 

529 ) 

530 

531 # total_unsupported_models (required int >= 0) 

532 errors.extend( 

533 _validate_int( 

534 data.get("total_unsupported_models"), 

535 "total_unsupported_models", 

536 min_value=0, 

537 ) 

538 ) 

539 

540 # gaps (required list of ArchitectureGap) 

541 gaps = data.get("gaps") 

542 errors.extend(_validate_list(gaps, "gaps")) 

543 if isinstance(gaps, list): 543 ↛ 547line 543 didn't jump to line 547 because the condition on line 543 was always true

544 for i, gap in enumerate(gaps): 

545 errors.extend(_validate_architecture_gap(gap, f"gaps[{i}]")) 

546 

547 return ValidationResult( 

548 valid=len(errors) == 0, 

549 errors=errors, 

550 schema_type="ArchitectureGapsReport", 

551 ) 

552 

553 

554def validate_verification_history(data: dict) -> ValidationResult: 

555 """Validate a VerificationHistory JSON object. 

556 

557 Args: 

558 data: Dictionary loaded from JSON to validate 

559 

560 Returns: 

561 ValidationResult with validation status and any errors 

562 """ 

563 errors = [] 

564 

565 if not isinstance(data, dict): 565 ↛ 566line 565 didn't jump to line 566 because the condition on line 565 was never true

566 return ValidationResult( 

567 valid=False, 

568 errors=[ 

569 ValidationError("", f"expected object at root, got {type(data).__name__}", data) 

570 ], 

571 schema_type="VerificationHistory", 

572 ) 

573 

574 # last_updated (optional datetime string) 

575 if "last_updated" in data and data["last_updated"] is not None: 575 ↛ 581line 575 didn't jump to line 581 because the condition on line 575 was always true

576 errors.extend( 

577 _validate_datetime_string(data["last_updated"], "last_updated", required=False) 

578 ) 

579 

580 # records (required list of VerificationRecord) 

581 records = data.get("records") 

582 errors.extend(_validate_list(records, "records")) 

583 if isinstance(records, list): 583 ↛ 587line 583 didn't jump to line 587 because the condition on line 583 was always true

584 for i, record in enumerate(records): 

585 errors.extend(_validate_verification_record(record, f"records[{i}]")) 

586 

587 return ValidationResult( 

588 valid=len(errors) == 0, 

589 errors=errors, 

590 schema_type="VerificationHistory", 

591 ) 

592 

593 

594def validate_json_schema(file_path: Path | str, schema_type: str | None = None) -> ValidationResult: 

595 """Validate a JSON file against its expected schema. 

596 

597 This function reads a JSON file and validates it against one of the model registry 

598 schemas. The schema type can be automatically inferred from the filename or 

599 explicitly specified. 

600 

601 Args: 

602 file_path: Path to the JSON file to validate 

603 schema_type: Schema type to validate against. If None, inferred from filename. 

604 Supported values: "supported_models", "architecture_gaps", "verification_history" 

605 

606 Returns: 

607 ValidationResult with validation status and any errors 

608 

609 Raises: 

610 FileNotFoundError: If the file does not exist 

611 json.JSONDecodeError: If the file is not valid JSON 

612 ValueError: If schema_type cannot be determined 

613 """ 

614 file_path = Path(file_path) 

615 

616 # Infer schema type from filename if not provided 

617 if schema_type is None: 617 ↛ 618line 617 didn't jump to line 618 because the condition on line 617 was never true

618 filename = file_path.stem.lower() 

619 if "supported_models" in filename or filename == "supported_models": 

620 schema_type = "supported_models" 

621 elif "architecture_gaps" in filename or filename == "architecture_gaps": 

622 schema_type = "architecture_gaps" 

623 elif "verification" in filename or filename == "verification_history": 

624 schema_type = "verification_history" 

625 else: 

626 raise ValueError( 

627 f"Cannot infer schema type from filename '{file_path.name}'. " 

628 "Please specify schema_type explicitly. " 

629 "Supported values: 'supported_models', 'architecture_gaps', 'verification_history'" 

630 ) 

631 

632 # Read and parse the JSON file 

633 with open(file_path) as f: 

634 data = json.load(f) 

635 

636 # Validate based on schema type 

637 if schema_type == "supported_models": 

638 return validate_supported_models_report(data) 

639 elif schema_type == "architecture_gaps": 

640 return validate_architecture_gaps_report(data) 

641 elif schema_type == "verification_history": 641 ↛ 644line 641 didn't jump to line 644 because the condition on line 641 was always true

642 return validate_verification_history(data) 

643 else: 

644 raise ValueError( 

645 f"Unknown schema_type: {schema_type}. " 

646 "Supported values: 'supported_models', 'architecture_gaps', 'verification_history'" 

647 ) 

648 

649 

650def validate_data_directory(data_dir: Path | str | None = None) -> dict[str, ValidationResult]: 

651 """Validate all JSON files in the data directory. 

652 

653 Validates supported_models.json, verification_history.json, and 

654 architecture_gaps.json. 

655 

656 Args: 

657 data_dir: Path to the data directory. If None, uses the default data directory. 

658 

659 Returns: 

660 Dictionary mapping filenames to their ValidationResults 

661 """ 

662 if data_dir is None: 

663 data_dir = Path(__file__).parent / "data" 

664 else: 

665 data_dir = Path(data_dir) 

666 

667 results = {} 

668 

669 # Validate supported_models.json 

670 supported_path = data_dir / "supported_models.json" 

671 if supported_path.exists(): 

672 try: 

673 results["supported_models.json"] = validate_json_schema( 

674 supported_path, "supported_models" 

675 ) 

676 except json.JSONDecodeError as e: 

677 results["supported_models.json"] = ValidationResult( 

678 valid=False, 

679 errors=[ValidationError("", f"Invalid JSON: {e}")], 

680 schema_type="supported_models", 

681 ) 

682 

683 # Validate architecture_gaps.json 

684 gaps_path = data_dir / "architecture_gaps.json" 

685 if gaps_path.exists(): 

686 try: 

687 results["architecture_gaps.json"] = validate_json_schema(gaps_path, "architecture_gaps") 

688 except json.JSONDecodeError as e: 

689 results["architecture_gaps.json"] = ValidationResult( 

690 valid=False, 

691 errors=[ValidationError("", f"Invalid JSON: {e}")], 

692 schema_type="architecture_gaps", 

693 ) 

694 

695 # Validate verification_history.json 

696 verification_path = data_dir / "verification_history.json" 

697 if verification_path.exists(): 

698 try: 

699 results["verification_history.json"] = validate_json_schema( 

700 verification_path, "verification_history" 

701 ) 

702 except json.JSONDecodeError as e: 

703 results["verification_history.json"] = ValidationResult( 

704 valid=False, 

705 errors=[ValidationError("", f"Invalid JSON: {e}")], 

706 schema_type="verification_history", 

707 ) 

708 

709 return results