{
  "schema_version": 1,
  "checked_at": "2026-08-19",
  "subject": "GLM-5.2 ColBERT retrieval and context-packing audit",
  "environment": {
    "container_image": "python@sha256:47ae396f09c1303b8653019811a8498470603d7ffefc29cb07c88f1f8cb3d19f",
    "python": "3.12.11",
    "torch": "2.13.0+cpu",
    "sentence_transformers": "6.0.0",
    "transformers": "5.15.0",
    "execution_network": "none",
    "device": "cpu"
  },
  "retriever": {
    "model_id": "mixedbread-ai/mxbai-edge-colbert-v0-17m",
    "revision": "21996dcf231e0b406c3342b374155e43d4960341",
    "similarity": "maxsim",
    "embedding_dimension": 48,
    "query_length": 48,
    "document_length": 512,
    "query_expansion": false,
    "corpus_documents": 8
  },
  "glm_tokenizer": {
    "model_id": "zai-org/GLM-5.2",
    "revision": "b4734de4facf877f85769a911abafc5283eab3d9",
    "model_max_length": 1048576,
    "note": "Counts cover the rendered evidence text only; chat-template and provider overhead are excluded."
  },
  "retrieval_evaluation": {
    "query_count": 6,
    "hit_at_1": 6,
    "queries": [
      {
        "id": "multi-vector-definition",
        "text": "Which Sentence Transformers component keeps one vector per token and scores with MaxSim?",
        "expected": "hf-v6-overview",
        "expected_rank": 1,
        "top_three": [
          {
            "rank": 1,
            "document_id": "hf-v6-overview",
            "score": 18.432579
          },
          {
            "rank": 2,
            "document_id": "hf-exact-and-semantic",
            "score": 17.976002
          },
          {
            "rank": 3,
            "document_id": "colbertv2-compression",
            "score": 17.959993
          }
        ],
        "query_vector_count": 19
      },
      {
        "id": "exact-plus-paraphrase",
        "text": "Which retrieval design can preserve exact API identifiers while still matching paraphrases?",
        "expected": "hf-exact-and-semantic",
        "expected_rank": 1,
        "top_three": [
          {
            "rank": 1,
            "document_id": "hf-exact-and-semantic",
            "score": 17.364098
          },
          {
            "rank": 2,
            "document_id": "glm52-context",
            "score": 17.120449
          },
          {
            "rank": 3,
            "document_id": "sbert-length-cap",
            "score": 17.079893
          }
        ],
        "query_vector_count": 18
      },
      {
        "id": "truncation-risk",
        "text": "Which checkpoint setting can silently remove the tail of a long passage before indexing?",
        "expected": "sbert-length-cap",
        "expected_rank": 1,
        "top_three": [
          {
            "rank": 1,
            "document_id": "sbert-length-cap",
            "score": 17.170918
          },
          {
            "rank": 2,
            "document_id": "glm52-context",
            "score": 16.999931
          },
          {
            "rank": 3,
            "document_id": "hf-v6-overview",
            "score": 16.940119
          }
        ],
        "query_vector_count": 18
      },
      {
        "id": "offline-documents",
        "text": "Can document representations be precomputed before a query arrives?",
        "expected": "colbert-precompute",
        "expected_rank": 1,
        "top_three": [
          {
            "rank": 1,
            "document_id": "colbert-precompute",
            "score": 14.616963
          },
          {
            "rank": 2,
            "document_id": "sbert-length-cap",
            "score": 14.448835
          },
          {
            "rank": 3,
            "document_id": "hf-v6-overview",
            "score": 14.364824
          }
        ],
        "query_vector_count": 15
      },
      {
        "id": "index-compression",
        "text": "What ColBERT method reduces the storage footprint of token vectors?",
        "expected": "colbertv2-compression",
        "expected_rank": 1,
        "top_three": [
          {
            "rank": 1,
            "document_id": "colbertv2-compression",
            "score": 14.409203
          },
          {
            "rank": 2,
            "document_id": "hf-v6-overview",
            "score": 14.393216
          },
          {
            "rank": 3,
            "document_id": "hf-exact-and-semantic",
            "score": 14.175205
          }
        ],
        "query_vector_count": 15
      },
      {
        "id": "architecture-boundary",
        "text": "Is GLM-5.2 IndexShare the same thing as an external ColBERT evidence retriever?",
        "expected": "glm52-indexshare",
        "expected_rank": 1,
        "top_three": [
          {
            "rank": 1,
            "document_id": "glm52-indexshare",
            "score": 24.221834
          },
          {
            "rank": 2,
            "document_id": "glm52-context",
            "score": 23.819492
          },
          {
            "rank": 3,
            "document_id": "glm52-mtp",
            "score": 23.811081
          }
        ],
        "query_vector_count": 25
      }
    ]
  },
  "context_pack": {
    "strategy": "decompose one compound reader need into two probes, take top-1 for each, deduplicate by source id, and retain source URLs",
    "probes": [
      {
        "text": "Which retrieval design preserves exact identifiers while still matching paraphrases?",
        "top_three": [
          {
            "rank": 1,
            "document_id": "hf-exact-and-semantic",
            "score": 15.468705
          },
          {
            "rank": 2,
            "document_id": "glm52-context",
            "score": 15.199931
          },
          {
            "rank": 3,
            "document_id": "colbertv2-compression",
            "score": 15.171385
          }
        ]
      },
      {
        "text": "Which checkpoint setting can discard the end of an overlong passage?",
        "top_three": [
          {
            "rank": 1,
            "document_id": "sbert-length-cap",
            "score": 15.251849
          },
          {
            "rank": 2,
            "document_id": "hf-v6-overview",
            "score": 15.145743
          },
          {
            "rank": 3,
            "document_id": "glm52-context",
            "score": 15.144522
          }
        ]
      }
    ],
    "selected_document_ids": [
      "hf-exact-and-semantic",
      "sbert-length-cap"
    ],
    "selected_source_urls": [
      "https://huggingface.co/blog/multi-vector-encoder",
      "https://www.sbert.net/docs/multi_vector_encoder/usage/usage.html"
    ],
    "selected_glm52_tokens": 128,
    "full_corpus_glm52_tokens": 527,
    "token_reduction_percent": 75.7,
    "selected_characters": 622,
    "full_corpus_characters": 2336,
    "selected_context_sha256": "e067bb30df05409218d22cbb7197e827c493c5fca6b14b2a4c77283c7280dbb9",
    "full_context_sha256": "603dee4540e6cf9147ded6ae47ac9ec7175a60d0dc22f6e906a3147ead64abec"
  },
  "index_arithmetic": {
    "document_vector_counts": [
      49,
      41,
      34,
      36,
      37,
      42,
      42,
      35
    ],
    "total_token_vectors": 316,
    "raw_float32_bytes": 60672,
    "raw_float32_kib": 59.25,
    "one_vector_per_document_bytes": 1536,
    "observed_raw_vector_count_ratio": 39.5,
    "boundary": "This is raw in-memory float32 arithmetic for the eight fixture passages, not a compressed production index benchmark."
  },
  "truncation_negative_control": {
    "configured_document_length": 512,
    "raw_token_counts": [
      3560,
      3560
    ],
    "encoded_vector_counts": [
      428,
      428
    ],
    "different_tail_markers": [
      "TAIL-SENTINEL-ALPHA",
      "TAIL-SENTINEL-OMEGA"
    ],
    "truncated_embeddings_identical": true
  },
  "assertions": [
    {
      "name": "sentence-transformers-version-pinned",
      "passed": true,
      "detail": "6.0.0"
    },
    {
      "name": "retriever-revision-pinned",
      "passed": true,
      "detail": "21996dcf231e0b406c3342b374155e43d4960341"
    },
    {
      "name": "glm-tokenizer-revision-pinned",
      "passed": true,
      "detail": "b4734de4facf877f85769a911abafc5283eab3d9"
    },
    {
      "name": "six-queries-hit-at-one",
      "passed": true,
      "detail": {
        "multi-vector-definition": 1,
        "exact-plus-paraphrase": 1,
        "truncation-risk": 1,
        "offline-documents": 1,
        "index-compression": 1,
        "architecture-boundary": 1
      }
    },
    {
      "name": "packing-selects-two-distinct-sources",
      "passed": true,
      "detail": [
        "hf-exact-and-semantic",
        "sbert-length-cap"
      ]
    },
    {
      "name": "selected-context-is-smaller",
      "passed": true,
      "detail": {
        "selected_tokens": 128,
        "full_tokens": 527
      }
    },
    {
      "name": "tail-negative-control-exceeds-cap",
      "passed": true,
      "detail": {
        "configured_document_length": 512,
        "raw_tokens": [
          3560,
          3560
        ]
      }
    },
    {
      "name": "different-tails-produce-identical-truncated-embeddings",
      "passed": true,
      "detail": {
        "embedding_shapes": [
          [
            428,
            48
          ],
          [
            428,
            48
          ]
        ],
        "identical": true
      }
    },
    {
      "name": "no-credentials-present-in-output",
      "passed": true,
      "detail": true
    }
  ],
  "summary": {
    "assertion_count": 9,
    "passed_assertion_count": 9,
    "failed_assertions": [],
    "real_glm_calls": 0,
    "external_api_calls_during_audit": 0,
    "credentials_used": false
  },
  "limits": [
    "The six expected passages were hand-labeled before scoring and form a small editorial fixture, not a retrieval benchmark.",
    "The English 17M-parameter checkpoint does not establish quality for another language, domain, corpus, or production index.",
    "MaxSim scores are used only for ranking within each query and are not compared across queries or checkpoints.",
    "Retrieval and context packing do not prove that GLM-5.2 will answer correctly, cite faithfully, or resist instructions contained in retrieved text.",
    "The GLM tokenizer count excludes chat-template, system-message, tool-schema, and provider-side overhead.",
    "The index arithmetic excludes metadata, ANN structures, quantization, compression, and runtime allocator overhead."
  ]
}