commit 2a10cd48c09928ba07b4e277a3a78117d563cd2e Author: KS Jannette Date: Wed Aug 26 21:58:34 2026 -0400 first commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e082f1a --- /dev/null +++ b/.gitignore @@ -0,0 +1,20 @@ +__pycache__/ +*.py[cod] +*$py.class + *.pyc +agents/ +bin/ +build/ +dist/ +downloads/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ + +.cache +.pytest_cache/ + +.venv/ \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..99d516f --- /dev/null +++ b/README.md @@ -0,0 +1,197 @@ +# [Sem_Cache](https://github.com/kjannette/semantic-cache-script) + +Sem_Cache is a tiny command-line tool written in [Python](https://www.python.org/) that +demonstrates semantic caching for Large Language Model (LLM) queries. It uses the +[Sentence Transformers](https://www.sbert.net/) library with [NumPy](https://numpy.org/) +to compute vector embeddings and [cosine similarity](https://en.wikipedia.org/wiki/Cosine_similarity) +between text inputs, determining whether an incoming query is similar enough to a +cached query to return a stored response instead of making a new LLM Application +Programming Interface (API) call. + +Released under the +[GPL Version 3](https://opensource.org/license/gpl-3-0). Initial release: August 2026. + +This project is intended for developers building LLM applications who want +to reduce API costs, latency, and redundant computations by caching semantically +equivalent queries. + +It can be used to quickly verify the reliability of a semantic caching layer by generating a cosine score using two test sentences, input as command line strings. + +The output can be diff’d against the behavior of the system under development for an instant “sanity check.” + +--- + +## How It Works + +If two queries mean the same thing, they should return the same answer. Rather than comparing strings character-by-character, it converts each query into a dense vector embedding using the [all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) model (a lightweight transformer that runs locally). It then calculates the cosine similarity between the cached query embedding and the new query embedding. + +If the similarity score meets or exceeds the threshold (default: 0.92), the system +declares a **cache hit** and would return the cached response. Otherwise, it declares +a **cache miss** and routes the request to the LLM. + +--- + +## Getting Started + +### Prerequisites + +- [Python](https://www.python.org/) 3.10 or higher +- [pip](https://pip.pypa.io/) (Python package installer) + +### Installation + +Clone the repository and set up a virtual environment: + +```bash +git clone https://github.com/kjannette/semantic-cache-script.git +cd semantic-cache-script +python -m venv .venv +source .venv/bin/activate # On Windows: .venv\Scripts\activate +``` + +Install the required dependencies: + +```bash +pip install numpy sentence-transformers +``` + +The first run will download the all-MiniLM-L6-v2 model (approximately 90 MB). + +### Usage + +Run the script from the command line: + +```bash +python sem_cache.py +``` + +The program will prompt you to enter two sentences: + +1. **Sentence 1**: The baseline cached query (simulating a query already in the cache) +2. **Sentence 2**: The new incoming query (simulating a user's new request) + +### Example Session + +``` +============================================================ +LOCAL SEMANTIC CACHE SIMULATOR +============================================================ +Loading local embedding model (all-MiniLM-L6-v2)... + +Model loaded successfully. +------------------------------------------------------------ +Enter Sentence 1 (The Baseline Cached Query): +> What is the weather like today? + +Enter Sentence 2 (The New Incoming Query): +> How's the weather today? + +Generating local embeddings and performing vector math... +------------------------------------------------------------ +RESULTS: +-> Calculated Cosine Similarity: 0.9412 +-> Target Safety Threshold: 0.9200 + +[CACHE HIT] Returning cached response. +============================================================ +``` + +In this example, the two queries are semantically equivalent, so the system returns +a cache hit with a similarity score of 0.9412. + +--- + +## Configuration + +The similarity threshold is set to 0.92 by default. This is a highly-conservative value, set to reduce false positives (treating dissimilar queries as matches). To adjust the threshold, +modify the `THRESHOLD` constant in `sem_cache.py`: + +```python +THRESHOLD = 0.92 +``` + +Lower values increase cache hit rates but risk returning incorrect cached responses. +Higher values reduce false positives but may miss valid semantic matches. + +--- + +## Project Structure + +``` +semantic-cache-script/ +├── sem_cache.py # Main application script +├── README.md # This file +├── .gitignore # Git ignore patterns +└── .venv/ # Python virtual environment (not tracked) +``` + +--- + +## Participation + +### Bug Reports + +Bug reports are accepted via [Git issues](https://github.com/kjannette/semantic-cache-script/issues). +Include the Python version, operating system, input sentences, and full error output +when reporting issues. + +### Pull Requests + +Pull requests are accepted for review. Project author makes no guarantee that +contributions will be merged. No Contributor License Agreement (CLA) is required. + +### Code Style + +This project follows [PEP 8](https://peps.python.org/pep-0008/) style guidelines. + +--- + +## Author + +- @ sjDev + +--- + +## Ideology + +This project does not have a formal Code of Conduct. + +Sem_Cache is a standalone, free software project. It is not associated with a +for-profit company or "open core" offering. The software runs locally and does +not transmit activity data off the device where it runs (the embedding model +executes on-device; no external API calls are made by the caching logic itself). + +--- + +## Roadmap / TO-DO + +(This README also serves as a development notebook.) + +### Planned Features + +- [ ] Print result to command line (completed in current version) +- [ ] Perform and output quantified metrics of what a cache hit conserves + +### Possible Metrics to Implement + +1. **Estimated completion tokens** — Calculate tokens saved by popular model + (assuming cache did not exist or cache misses) +2. **Latency** — Measure the API call latency to the LLM that would be avoided + on cache hit +3. **Rate limit / quota impact** — Theoretical calculation of quota preservation + (low priority) +4. **Cost of redundant embedding** — Quantify savings from not re-embedding + identical strings across runs, retries, and re-indexes +5. **Provider outage resilience** — Document how cached responses provide + continuity when upstream services are unavailable +6. **Reduction of nondeterminism** — Address unreliable tests and unreproducible + bugs by enabling diff comparisons between runs when model output varies +7. **Concurrency waste avoidance** — Quantify savings when N users submit + identical queries (e.g., 50 users click the same button, ask the same + question; without cache, cost = 50x) + +--- + +## License + +This softeare is released under the [GNU General Public License Version 3](https://opensource.org/license/gpl-3-0). diff --git a/sem_cache.py b/sem_cache.py new file mode 100644 index 0000000..4f31b7c --- /dev/null +++ b/sem_cache.py @@ -0,0 +1,60 @@ +import sys +import numpy as np +from sentence_transformers import SentenceTransformer + +def main(): + print("=" * 60) + print("LOCAL SEMANTIC CACHE SIMULATOR") + print("=" * 60) + + # Initialize lightweight local embedding model + print("Loading local embedding model (all-MiniLM-L6-v2)...") + try: + model = SentenceTransformer("all-MiniLM-L6-v2") + except Exception as e: + print(f"Error loading model: {e}") + sys.exit(1) + + print("\nModel loaded successfully.") + print("-" * 60) + + # Enforce strict threshold to avoid false-positives + THRESHOLD = 0.92 + + # User input + sentence_1 = input("Enter Sentence 1 (The Baseline Cached Query):\n> ").strip() + sentence_2 = input("\nEnter Sentence 2 (The New Incoming Query):\n> ").strip() + + if not sentence_1 or not sentence_2: + print("\n[Error] Both sentences must contain text.") + sys.exit(1) + + print("\nGenerating local embeddings and performing vector math...") + + # Generate embeddings (Returns 1D NumPy array for each string) + emb1 = model.encode(sentence_1) + emb2 = model.encode(sentence_2) + + # Calculate Cosine Similarity w NumPy vector operations + dot_product = np.dot(emb1, emb2) + norm_1 = np.linalg.norm(emb1) + norm_2 = np.linalg.norm(emb2) + + similarity_score = dot_product / (norm_1 * norm_2) + + print("-" * 60) + print(f"RESULTS:") + print(f"-> Calculated Cosine Similarity: {similarity_score:.4f}") + print(f"-> Target Safety Threshold: {THRESHOLD:.4f}") + + # Evaluate Cache Efficacy + if similarity_score >= THRESHOLD: + print("\n[CACHE HIT] Returning cached response.") + else: + print("\n[CACHE MISS] Score below threshold. Request routed to LLM.") + print("=" * 60) + +if __name__ == "__main__": + main() + +# To do: see README.md \ No newline at end of file