documents = [
    'the product quality is excellent and shipping was fast',
    'terrible experience, the item arrived broken and late',
    'great value for the price, very happy with this purchase',
    'awful quality, would not recommend this product at all',
    'fast shipping and excellent customer service overall',
    'broken item, poor quality, very disappointing purchase'
]
labels = [1, 0, 1, 0, 1, 0]
print(len(documents), len(labels))

from sklearn.feature_extraction.text import CountVectorizer
count_vec = CountVectorizer()
X_counts = count_vec.fit_transform(documents)
print(X_counts.shape)
print(X_counts.toarray()[0][:10])

print(count_vec.vocabulary_['quality'])
feature_names = count_vec.get_feature_names_out()
print(len(feature_names))
print(sorted(feature_names)[:10])

filtered_vec = CountVectorizer(stop_words='english', min_df=1, max_df=0.9)
X_filtered = filtered_vec.fit_transform(documents)
print(X_filtered.shape[1], X_counts.shape[1])
print('the' in filtered_vec.vocabulary_)

bigram_vec = CountVectorizer(stop_words='english', ngram_range=(1, 2))
X_bigram = bigram_vec.fit_transform(documents)
bigram_features = [f for f in bigram_vec.get_feature_names_out() if ' ' in f]
print(bigram_features[:5])
print(X_bigram.shape[1], X_filtered.shape[1])

from sklearn.feature_extraction.text import TfidfVectorizer
tfidf_vec = TfidfVectorizer(stop_words='english')
X_tfidf = tfidf_vec.fit_transform(documents)
print(X_tfidf.shape)
print(X_tfidf.toarray()[0].round(3)[:8])

quality_idx = filtered_vec.vocabulary_['quality']
tfidf_quality_idx = tfidf_vec.vocabulary_['quality']
print('count weight:', X_filtered.toarray()[:, quality_idx])
print('tfidf weight:', X_tfidf.toarray()[:, tfidf_quality_idx].round(3))

new_reviews = ['excellent quality and fast shipping', 'this is a completely novel gibberish sentence']
X_new = tfidf_vec.transform(new_reviews)
print(X_new.shape)
print(round(X_new[1].sum(), 3))

from sklearn.pipeline import Pipeline
from sklearn.linear_model import LogisticRegression
text_pipe = Pipeline([
    ('tfidf', TfidfVectorizer(stop_words='english')),
    ('model', LogisticRegression(max_iter=1000))
])
text_pipe.fit(documents, labels)
print(text_pipe.predict(['fast shipping and excellent quality']))

def build_text_classifier(ngram_range=(1, 1), max_features=None):
    return Pipeline([
        ('tfidf', TfidfVectorizer(stop_words='english', ngram_range=ngram_range, max_features=max_features)),
        ('model', LogisticRegression(max_iter=1000))
    ])
clf = build_text_classifier(ngram_range=(1, 2))
clf.fit(documents, labels)
print(clf.predict(['terrible broken item, very poor']))