# Current prediction
score = dot_product(target_vector, context_vector)
prediction = sigmoid(score)

# Error calculation
error = actual_label - prediction

# Coordinate adjustment (THE MAGIC!)
target_vector += learning_rate * error * context_vector
context_vector += learning_rate * error * target_vector

for epoch in range(epochs):
    for target_word, context_word in training_pairs:
        # Generate positive and negative samples
        samples = generate_samples(target_word, context_word)
        
        for target, context, label in samples:
            # Make prediction
            prediction = neural_network(target, context)
            
            # Calculate error
            error = label - prediction
            
            # Adjust coordinates
            update_vectors(target, context, error)

similarity = dot_product(word1_vector, word2_vector) / 
            (magnitude(word1_vector) * magnitude(word2_vector))

result_vector = word_vector('king') - word_vector('man') + word_vector('woman')
closest_word = find_most_similar_to(result_vector)
# Returns: "queen"

# Get the code
git clone your-repository
pip install numpy matplotlib scikit-learn

# Download a dataset from Kaggle
kaggle datasets download -d snap/amazon-fine-food-reviews

# Initialize the model
model = Word2VecTrainer(vector_size=100, epochs=5)

# Load your text data
sentences = load_and_preprocess_your_data()

# Train (this is where the magic happens!)
model.train(sentences)

# Test similarities
print(model.similarity('good', 'great'))
print(model.most_similar('delicious'))

# Try analogies  
result = model.analogy('man', 'woman', 'king')
print(f"man:woman :: king:{result}")

# Plot word clusters
words = ['good', 'great', 'excellent', 'bad', 'terrible', 'awful']
visualize_embeddings(model, words)

import numpy as np
import re
from collections import defaultdict, Counter
import pickle
import matplotlib.pyplot as plt
from sklearn.manifold import TSNE
import logging

# Set up logging to watch the magic happen
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(message)s')
logger = logging.getLogger(__name__)

class Word2VecTrainer:
    """
    The REAL Word2Vec algorithm - Skip-gram with Negative Sampling
    This is what Google, Facebook, and OpenAI use (with optimizations)
    """
    
    def __init__(self, 
                 vector_size=100,      # Dimensionality of word vectors
                 window_size=5,        # Context window size
                 min_count=5,          # Ignore words with fewer occurrences
                 negative_samples=5,   # Number of negative samples
                 learning_rate=0.025,  # Learning rate
                 epochs=5):            # Number of training epochs
        
        self.vector_size = vector_size
        self.window_size = window_size
        self.min_count = min_count
        self.negative_samples = negative_samples
        self.learning_rate = learning_rate
        self.epochs = epochs
        
        # These will store our learned embeddings
        self.word2idx = {}
        self.idx2word = {}
        self.word_vectors = None
        self.context_vectors = None
        self.word_counts = Counter()
        
    def preprocess_text(self, text):
        """
        Clean and tokenize text - this is where the journey begins!
        """
        # Convert to lowercase and remove special characters
        text = re.sub(r'[^a-zA-Z\s]', '', text.lower())
        
        # Split into words
        words = text.split()
        
        # Remove very short words
        words = [word for word in words if len(word) > 2]
        
        return words
    
    def build_vocabulary(self, sentences):
        """
        Build vocabulary from text corpus
        This determines which words get embeddings
        """
        logger.info("Building vocabulary from corpus...")
        
        # Count word frequencies
        for sentence in sentences:
            for word in sentence:
                self.word_counts[word] += 1
        
        # Filter out rare words (they don't have enough context to learn well)
        vocab_words = [word for word, count in self.word_counts.items() 
                      if count >= self.min_count]
        
        # Create word-to-index mappings
        self.word2idx = {word: idx for idx, word in enumerate(vocab_words)}
        self.idx2word = {idx: word for word, idx in self.word2idx.items()}
        
        vocab_size = len(self.word2idx)
        logger.info(f"Vocabulary size: {vocab_size} words")
        
        # Initialize word vectors randomly (this is where coordinates start!)
        # Each word gets a random starting position in high-dimensional space
        self.word_vectors = np.random.uniform(-0.5, 0.5, (vocab_size, self.vector_size))
        self.context_vectors = np.random.uniform(-0.5, 0.5, (vocab_size, self.vector_size))
        
        # Normalize vectors
        self.word_vectors = self.word_vectors / np.linalg.norm(self.word_vectors, axis=1, keepdims=True)
        self.context_vectors = self.context_vectors / np.linalg.norm(self.context_vectors, axis=1, keepdims=True)
        
        return vocab_size
    
    def generate_training_pairs(self, sentences):
        """
        Generate (target_word, context_word) pairs for training
        This is where the algorithm learns "which words appear together"
        """
        training_pairs = []
        
        for sentence in sentences:
            # Convert words to indices
            word_indices = [self.word2idx[word] for word in sentence 
                          if word in self.word2idx]
            
            # For each word in the sentence
            for i, target_word_idx in enumerate(word_indices):
                # Look at surrounding words within window
                start = max(0, i - self.window_size)
                end = min(len(word_indices), i + self.window_size + 1)
                
                for j in range(start, end):
                    if i != j:  # Don't pair word with itself
                        context_word_idx = word_indices[j]
                        training_pairs.append((target_word_idx, context_word_idx))
        
        logger.info(f"Generated {len(training_pairs)} training pairs")
        return training_pairs
    
    def negative_sampling(self, target_word_idx, context_word_idx):
        """
        Generate negative samples for training
        This teaches the model what words DON'T go together
        """
        # Get positive pair
        positive_pairs = [(target_word_idx, context_word_idx, 1)]
        
        # Generate negative pairs (words that don't actually appear together)
        negative_pairs = []
        vocab_size = len(self.word2idx)
        
        for _ in range(self.negative_samples):
            # Randomly sample a word that's NOT the actual context
            negative_context = np.random.randint(0, vocab_size)
            while negative_context == context_word_idx:
                negative_context = np.random.randint(0, vocab_size)
            
            negative_pairs.append((target_word_idx, negative_context, 0))
        
        return positive_pairs + negative_pairs
    
    def sigmoid(self, x):
        """Sigmoid activation function"""
        # Clip to prevent overflow
        x = np.clip(x, -500, 500)
        return 1 / (1 + np.exp(-x))
    
    def train_on_pair(self, target_idx, context_idx, label):
        """
        THE CORE LEARNING ALGORITHM!
        This is where coordinates get adjusted based on prediction errors
        """
        # Get current vectors
        target_vector = self.word_vectors[target_idx]
        context_vector = self.context_vectors[context_idx]
        
        # Calculate prediction (dot product + sigmoid)
        score = np.dot(target_vector, context_vector)
        prediction = self.sigmoid(score)
        
        # Calculate error (how wrong were we?)
        error = label - prediction
        
        # Calculate gradients (which direction to move vectors)
        gradient = error * self.learning_rate
        
        # THIS IS THE MAGIC: Adjust vectors to reduce error
        # If words should be together (label=1), move them closer
        # If words shouldn't be together (label=0), move them apart
        target_update = gradient * context_vector
        context_update = gradient * target_vector
        
        # Update the vectors (this is where coordinates change!)
        self.word_vectors[target_idx] += target_update
        self.context_vectors[context_idx] += context_update
        
        return abs(error)
    
    def train(self, sentences):
        """
        Main training loop - this is where the magic happens!
        """
        logger.info("Starting Word2Vec training...")
        
        # Build vocabulary and initialize vectors
        vocab_size = self.build_vocabulary(sentences)
        
        # Generate training pairs
        training_pairs = self.generate_training_pairs(sentences)
        
        # Training loop
        for epoch in range(self.epochs):
            logger.info(f"Epoch {epoch + 1}/{self.epochs}")
            
            total_error = 0
            pair_count = 0
            
            # Shuffle training pairs for better learning
            np.random.shuffle(training_pairs)
            
            # Process each training pair
            for target_idx, context_idx in training_pairs:
                # Generate positive and negative samples
                samples = self.negative_sampling(target_idx, context_idx)
                
                # Train on each sample
                for target, context, label in samples:
                    error = self.train_on_pair(target, context, label)
                    total_error += error
                    pair_count += 1
                
                # Log progress
                if pair_count % 100000 == 0:
                    avg_error = total_error / pair_count
                    logger.info(f"  Processed {pair_count} pairs, avg error: {avg_error:.4f}")
            
            # Decay learning rate
            self.learning_rate *= 0.9
            
            avg_epoch_error = total_error / pair_count
            logger.info(f"Epoch {epoch + 1} completed. Average error: {avg_epoch_error:.4f}")
        
        logger.info("Training completed!")
    
    def get_word_vector(self, word):
        """Get the learned vector for a word"""
        if word in self.word2idx:
            idx = self.word2idx[word]
            return self.word_vectors[idx]
        else:
            return None
    
    def similarity(self, word1, word2):
        """
        Calculate cosine similarity between two words
        This is how we measure if the algorithm learned meaningful relationships!
        """
        vec1 = self.get_word_vector(word1)
        vec2 = self.get_word_vector(word2)
        
        if vec1 is not None and vec2 is not None:
            # Cosine similarity
            cos_sim = np.dot(vec1, vec2) / (np.linalg.norm(vec1) * np.linalg.norm(vec2))
            return cos_sim
        else:
            return None
    
    def most_similar(self, word, top_n=5):
        """
        Find words most similar to given word
        This shows what the algorithm learned!
        """
        word_vec = self.get_word_vector(word)
        if word_vec is None:
            return []
        
        similarities = []
        for other_word, idx in self.word2idx.items():
            if other_word != word:
                other_vec = self.word_vectors[idx]
                cos_sim = np.dot(word_vec, other_vec) / (
                    np.linalg.norm(word_vec) * np.linalg.norm(other_vec)
                )
                similarities.append((other_word, cos_sim))
        
        # Sort by similarity
        similarities.sort(key=lambda x: x[1], reverse=True)
        return similarities[:top_n]
    
    def analogy(self, word_a, word_b, word_c):
        """
        Solve analogies: word_a is to word_b as word_c is to ?
        This is the famous King - Man + Woman = Queen calculation!
        """
        vec_a = self.get_word_vector(word_a)
        vec_b = self.get_word_vector(word_b)
        vec_c = self.get_word_vector(word_c)
        
        if None in [vec_a, vec_b, vec_c]:
            return None
        
        # Calculate: vec_b - vec_a + vec_c
        result_vec = vec_b - vec_a + vec_c
        
        # Find most similar word to result_vec
        best_word = None
        best_similarity = -1
        
        for word, idx in self.word2idx.items():
            if word not in [word_a, word_b, word_c]:
                word_vec = self.word_vectors[idx]
                similarity = np.dot(result_vec, word_vec) / (
                    np.linalg.norm(result_vec) * np.linalg.norm(word_vec)
                )
                if similarity > best_similarity:
                    best_similarity = similarity
                    best_word = word
        
        return best_word, best_similarity
    
    def save_model(self, filename):
        """Save the trained model"""
        model_data = {
            'word2idx': self.word2idx,
            'idx2word': self.idx2word,
            'word_vectors': self.word_vectors,
            'context_vectors': self.context_vectors,
            'vector_size': self.vector_size
        }
        
        with open(filename, 'wb') as f:
            pickle.dump(model_data, f)
        
        logger.info(f"Model saved to {filename}")
    
    def load_model(self, filename):
        """Load a trained model"""
        with open(filename, 'rb') as f:
            model_data = pickle.load(f)
        
        self.word2idx = model_data['word2idx']
        self.idx2word = model_data['idx2word']
        self.word_vectors = model_data['word_vectors']
        self.context_vectors = model_data['context_vectors']
        self.vector_size = model_data['vector_size']
        
        logger.info(f"Model loaded from {filename}")

def visualize_embeddings(model, words_to_plot):
    """
    Visualize word embeddings in 2D using t-SNE
    This shows you the learned coordinate space!
    """
    vectors = []
    labels = []
    
    for word in words_to_plot:
        vec = model.get_word_vector(word)
        if vec is not None:
            vectors.append(vec)
            labels.append(word)
    
    if len(vectors) < 2:
        print("Not enough words found in vocabulary")
        return
    
    # Reduce dimensionality to 2D for plotting
    vectors = np.array(vectors)
    tsne = TSNE(n_components=2, random_state=42)
    vectors_2d = tsne.fit_transform(vectors)
    
    # Plot
    plt.figure(figsize=(12, 8))
    plt.scatter(vectors_2d[:, 0], vectors_2d[:, 1])
    
    for i, label in enumerate(labels):
        plt.annotate(label, (vectors_2d[i, 0], vectors_2d[i, 1]))
    
    plt.title("Word Embeddings Visualization")
    plt.xlabel("Dimension 1")
    plt.ylabel("Dimension 2")
    plt.grid(True)
    plt.show()

# DEMO: How to use this million-dollar algorithm
def demo_word2vec():
    """
    Demo showing the algorithm in action
    Replace this with your Kaggle dataset!
    """
    
    # Sample text (replace with your massive dataset!)
    sample_texts = [
        "the king ruled the kingdom with wisdom and strength",
        "the queen wore a beautiful crown made of gold",
        "the man walked to work every morning",
        "the woman drove her car to the office",
        "kings and queens live in magnificent palaces",
        "men and women work together in companies",
        "the royal king and his elegant queen attended the ceremony",
        "ordinary men and women gathered in the town square",
        "the wise king made decisions for his people",
        "the graceful queen danced at the royal ball",
        "the tall man carried heavy boxes",
        "the smart woman solved complex problems",
        "ancient kings built massive castles",
        "medieval queens wore expensive jewelry",
        "strong men lifted heavy weights",
        "intelligent women led important meetings"
    ] * 1000  # Repeat to have more training data
    
    # Initialize and train the model
    model = Word2VecTrainer(vector_size=50, epochs=10)
    
    # Preprocess texts
    sentences = [model.preprocess_text(text) for text in sample_texts]
    
    # Train the model (this is where coordinates are learned!)
    model.train(sentences)
    
    # Test the learned relationships
    print("\n=== LEARNED SIMILARITIES ===")
    test_words = ['king', 'queen', 'man', 'woman']
    
    for word in test_words:
        if word in model.word2idx:
            similar = model.most_similar(word, top_n=3)
            print(f"{word}: {similar}")
    
    print("\n=== SIMILARITY SCORES ===")
    pairs = [('king', 'queen'), ('king', 'man'), ('man', 'woman'), ('king', 'woman')]
    for word1, word2 in pairs:
        sim = model.similarity(word1, word2)
        if sim is not None:
            print(f"{word1} ↔ {word2}: {sim:.3f}")
    
    print("\n=== ANALOGY TEST ===")
    result = model.analogy('king', 'man', 'woman')
    if result:
        word, score = result
        print(f"king - man + woman = {word} (confidence: {score:.3f})")
    
    # Visualize if possible
    words_to_plot = ['king', 'queen', 'man', 'woman', 'palace', 'crown', 'work', 'office']
    try:
        visualize_embeddings(model, words_to_plot)
    except ImportError:
        print("Install matplotlib and scikit-learn for visualization")
    
    return model

if __name__ == "__main__":
    print("🚀 STARTING MILLION DOLLAR WORD2VEC ALGORITHM 🚀")
    print("=" * 60)
    
    # Run the demo
    trained_model = demo_word2vec()
    
    print("\n🎉 TRAINING COMPLETE! 🎉")
    print("The algorithm has learned word coordinates from scratch!")
    print("No hand-coding - pure machine learning discovery!")

pip install numpy matplotlib scikit-learn kaggle

# Install Kaggle CLI if not already done
pip install kaggle

# Create Kaggle API credentials
# 1. Go to kaggle.com → Account → Create New API Token
# 2. Download kaggle.json file
# 3. Place it in ~/.kaggle/ (Linux/Mac) or C:\Users\{username}\.kaggle\ (Windows)
# 4. Set permissions: chmod 600 ~/.kaggle/kaggle.json

# Option A: Amazon Food Reviews (500MB)
kaggle datasets download -d snap/amazon-fine-food-reviews

# Option B: News Category Dataset (200MB)  
kaggle datasets download -d rmisra/news-category-dataset

# Option C: Wikipedia Articles (2GB+ - for serious testing)
kaggle datasets download -d jkkphys/english-wikipedia-articles-20170820-sqlite

# Extract downloaded files
unzip amazon-fine-food-reviews.zip

# Add this to your run_word2vec.py
import pandas as pd

def load_amazon_reviews(file_path):
    """Load and preprocess Amazon reviews dataset"""
    df = pd.read_csv(file_path)
    
    # Extract text column (adjust column name as needed)
    texts = df['Text'].dropna().tolist()  # or 'Summary' column
    
    # Limit to first 10,000 reviews for testing (remove for full dataset)
    texts = texts[:10000]
    
    print(f"Loaded {len(texts)} text samples")
    return texts

def load_news_dataset(file_path):
    """Load and preprocess news dataset"""
    df = pd.read_json(file_path, lines=True)
    
    # Combine headline and short description
    texts = (df['headline'] + ' ' + df['short_description']).dropna().tolist()
    
    texts = texts[:10000]  # Limit for testing
    print(f"Loaded {len(texts)} news articles")
    return texts

# Create test_run.py
from word2vec_trainer import Word2VecTrainer, demo_word2vec

# Quick test with sample data
print("=== QUICK TEST ===")
model = demo_word2vec()

# Create real_data_run.py
from word2vec_trainer import Word2VecTrainer
import pandas as pd

# Load your chosen dataset
texts = load_amazon_reviews('Reviews.csv')  # or your dataset file

# Initialize model with appropriate parameters
model = Word2VecTrainer(
    vector_size=100,    # Start with 100 dimensions
    window_size=5,      # 5 words context window
    min_count=5,        # Ignore words appearing less than 5 times
    negative_samples=5, # 5 negative samples per positive
    learning_rate=0.025,
    epochs=5            # Start with 5 epochs
)

# Preprocess all texts
print("Preprocessing texts...")
sentences = [model.preprocess_text(text) for text in texts]

# Filter out empty sentences
sentences = [sent for sent in sentences if len(sent) > 3]
print(f"Ready to train on {len(sentences)} sentences")

# Train the model (this will take time!)
print("Starting training... (this may take 30+ minutes)")
model.train(sentences)

# Save the trained model
model.save_model('my_word2vec_model.pkl')
print("Model saved!")

# Create evaluate_model.py
from word2vec_trainer import Word2VecTrainer

# Load your trained model
model = Word2VecTrainer()
model.load_model('my_word2vec_model.pkl')

# Test word similarities
print("=== SIMILARITY TESTS ===")
test_pairs = [
    ('good', 'great'),
    ('food', 'restaurant'), 
    ('delicious', 'tasty'),
    ('bad', 'terrible')
]

for word1, word2 in test_pairs:
    sim = model.similarity(word1, word2)
    if sim is not None:
        print(f"{word1} ↔ {word2}: {sim:.3f}")
    else:
        print(f"One of {word1}/{word2} not in vocabulary")

# Find similar words
print("\n=== MOST SIMILAR WORDS ===")
test_words = ['delicious', 'restaurant', 'good', 'bad']
for word in test_words:
    similar = model.most_similar(word, top_n=5)
    if similar:
        print(f"{word}: {similar}")

# Test analogies
print("\n=== ANALOGY TESTS ===")
analogy_tests = [
    ('good', 'better', 'bad'),      # good:better :: bad:?
    ('restaurant', 'food', 'hotel'), # restaurant:food :: hotel:?
]

for a, b, c in analogy_tests:
    result = model.analogy(a, b, c)
    if result:
        word, score = result
        print(f"{a}:{b} :: {c}:{word} (confidence: {score:.3f})")

# Solution: Reduce dataset size or parameters
texts = texts[:5000]  # Use fewer texts
model = Word2VecTrainer(vector_size=50, epochs=3)  # Smaller model

# Solution: Check if word exists before testing
if 'your_word' in model.word2idx:
    # Test the word
else:
    print("Word not found - try a more common word")

# Solution: Start smaller and scale up
model = Word2VecTrainer(
    vector_size=50,     # Reduced from 100
    epochs=3,           # Reduced from 5
    min_count=10        # Higher threshold = smaller vocabulary
)

# Production parameters for larger datasets
model = Word2VecTrainer(
    vector_size=300,     # More expressive vectors
    window_size=10,      # Larger context window
    min_count=3,         # Keep more words
    epochs=10,           # More training iterations
    learning_rate=0.01   # Fine-tuned learning rate
)