In [13]:
import setup_env
--------------------------------------------------------------------------------
=== Hardware Acceleration ===
PyTorch version: 2.9.0a0+145a3a7bda.nv25.10
Using NVIDIA GPU (CUDA)
CUDA version: 13.0
GPU name: NVIDIA GeForce RTX 5070 Ti
GPU count: 1
Total GPU memory: 15.92 GB
Allocated memory: 0.06 GB
Free memory: 15.86 GB
Device: cuda
=== Matplotlib Settings ===
✅ Font: NanumGothic
=== System Info ===
OS: Ubuntu 24.04.3 LTS (Noble Numbat)
Kernel: 6.6.87.2-microsoft-standard-WSL2
Architecture: x86_64
Python: 3.12.3
Working directory: /workspace/ai-deeplearning/tutorial
=== Library Versions ===
NumPy: 2.1.0
Pandas: 3.0.0
Matplotlib: 3.10.7
Scikit-learn: 1.7.2
OpenCV: Not installed → !pip install -q opencv-python
Pillow: 12.0.0
Seaborn: 0.13.2
TensorFlow: Not installed → !pip install -q tensorflow
Transformers: 4.40.1
TorchVision: 0.24.0a0+094e7af5
=== Environment setup completed ===
--------------------------------------------------------------------------------
=== Visualizing Test Plot (Wide View) ===
=== GPU Usage Code Snippet === Device set to: cuda ---------------------------------------- # 아래 코드를 복사해서 모델과 데이터를 GPU로 보내세요: model = YourModel().to(device) data = data.to(device) ---------------------------------------- === Environment setup completed === --------------------------------------------------------------------------------
In [2]:
import os
# 디렉토리 생성
os.makedirs('./data/ag_news', exist_ok=True)
# 이미 다운받은게 있으면 스킵
base_url = "https://raw.githubusercontent.com/mhjabreel/CharCnn_Keras/master/data/ag_news_csv"
for filename in ['train.csv', 'test.csv']:
filepath = f'./data/ag_news/{filename}'
if os.path.exists(filepath):
print(f"✅ {filename} 이미 존재 → 스킵")
else:
print(f"⬇️ {filename} 다운로드 중...")
os.system(f'wget -P ./data/ag_news {base_url}/{filename}')
print(f"✅ {filename} 다운로드 완료")
✅ train.csv 이미 존재 → 스킵 ✅ test.csv 이미 존재 → 스킵
In [3]:
import pandas as pd
df_train = pd.read_csv('./data/ag_news/train.csv', header=None)
df_test = pd.read_csv('./data/ag_news/test.csv', header=None)
print("컬럼 수:", df_train.shape)
print("\n샘플 데이터:")
print(df_train.head(3))
print("\n결측값:", df_train.isnull().sum().tolist())
print("레이블 분포:", df_train[0].value_counts().to_dict())
컬럼 수: (120000, 3)
샘플 데이터:
0 1 \
0 3 Wall St. Bears Claw Back Into the Black (Reuters)
1 3 Carlyle Looks Toward Commercial Aerospace (Reuters)
2 3 Oil and Economy Cloud Stocks' Outlook (Reuters)
2
0 Reuters - Short-sellers, Wall Street's dwindling\band of ultra-cynics, are seeing green again.
1 Reuters - Private investment firm Carlyle Group,\which has a reputation for making well-timed and occasionally\controversial plays in the defense industry, has quietly placed\its bets on another part of the market.
2 Reuters - Soaring crude prices plus worries\about the economy and the outlook for earnings are expected to\hang over the stock market next week during the depth of the\summer doldrums.
결측값: [0, 0, 0]
레이블 분포: {3: 30000, 4: 30000, 2: 30000, 1: 30000}
In [4]:
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
df_train = pd.read_csv('./data/ag_news/train.csv', header=None, names=['label', 'title', 'body'])
df_test = pd.read_csv('./data/ag_news/test.csv', header=None, names=['label', 'title', 'body'])
# RoBERTa는 간단한 전처리만
def preprocess(title, body):
text = str(title) + ' ' + str(body)
text = text.strip() # 앞뒤 공백만 제거
return text
df_train['text'] = df_train.apply(lambda r: preprocess(r['title'], r['body']), axis=1)
df_test['text'] = df_test.apply(lambda r: preprocess(r['title'], r['body']), axis=1)
df_train['label'] = df_train['label'] - 1
df_test['label'] = df_test['label'] - 1
df_tr, df_val = train_test_split(df_train, test_size=0.2, random_state=42, stratify=df_train['label'])
label_names = ['World', 'Sports', 'Business', 'Sci/Tech']
print(f"훈련셋: {len(df_tr):,}개")
print(f"검증셋: {len(df_val):,}개")
print(f"테스트셋: {len(df_test):,}개")
# ── 1. 데이터셋 크기 비교 ──
fig, axes = plt.subplots(1, 3, figsize=(15, 4))
splits = {'Train': df_tr, 'Val': df_val, 'Test': df_test}
colors = ['#4C72B0', '#DD8452', '#55A868']
for ax, (name, df), color in zip(axes, splits.items(), colors):
counts = df['label'].value_counts().sort_index()
ax.bar([label_names[i] for i in counts.index], counts.values, color=color, edgecolor='white')
ax.set_title(f'{name} ({len(df):,}개)')
ax.set_ylabel('개수')
for i, v in enumerate(counts.values):
ax.text(i, v + 100, str(v), ha='center', fontweight='bold', fontsize=9)
plt.suptitle('데이터셋 분할 및 레이블 분포', fontsize=13, fontweight='bold')
plt.tight_layout()
plt.show()
# ── 2. 텍스트 길이 분포 ──
for df, name in [(df_tr, 'Train'), (df_val, 'Val'), (df_test, 'Test')]:
df['text_len'] = df['text'].apply(lambda x: len(x.split()))
fig, axes = plt.subplots(1, 3, figsize=(15, 4))
for ax, (name, df), color in zip(axes, splits.items(), colors):
ax.hist(df['text_len'], bins=50, color=color, edgecolor='white')
ax.axvline(df['text_len'].mean(), color='red', linestyle='--', label=f'평균: {df["text_len"].mean():.0f}')
ax.set_title(f'{name} 텍스트 길이 분포')
ax.set_xlabel('단어 수')
ax.set_ylabel('빈도')
ax.legend()
plt.suptitle('텍스트 길이 분포', fontsize=13, fontweight='bold')
plt.tight_layout()
plt.show()
# ── 3. 데이터셋 비율 파이차트 ──
fig, ax = plt.subplots(figsize=(6, 6))
sizes = [len(df_tr), len(df_val), len(df_test)]
labels = [f'Train\n{len(df_tr):,}개', f'Val\n{len(df_val):,}개', f'Test\n{len(df_test):,}개']
ax.pie(sizes, labels=labels, colors=colors, autopct='%1.1f%%',
startangle=90, wedgeprops=dict(edgecolor='white', linewidth=2))
ax.set_title('데이터셋 분할 비율', fontsize=13, fontweight='bold')
plt.show()
훈련셋: 96,000개 검증셋: 24,000개 테스트셋: 7,600개
In [5]:
from collections import Counter
# text 컬럼 사용 (불용어 유지!)
counter_rnn = Counter()
for text in df_tr['text']:
counter_rnn.update(text.split())
MAX_VOCAB = 20000
vocab = {'<PAD>': 0, '<UNK>': 1}
for word, _ in counter_rnn.most_common(MAX_VOCAB - 2):
vocab[word] = len(vocab)
print(f"전체 고유 단어 수: {len(counter_rnn):,}")
print(f"사전 크기: {len(vocab):,}")
전체 고유 단어 수: 167,289 사전 크기: 20,000
In [6]:
# 텍스트 → 인덱스 변환 및 패딩
MAX_LEN = 64
def text_to_ids(text, vocab, max_len):
tokens = text.split()[:max_len]
ids = [vocab.get(t, 1) for t in tokens] # 없으면 <UNK>=1
ids += [0] * (max_len - len(ids)) # <PAD>=0 으로 패딩
return ids
# 변환 확인
sample = text_to_ids(df_tr['text'].iloc[0], vocab, MAX_LEN)
print(f"원본: {df_tr['text'].iloc[0][:60]}...")
print(f"변환: {sample[:10]}...")
print(f"길이: {len(sample)}")
원본: Clijsters Unsure About Latest Injury, Says Hewitt TOKYO (Re... 변환: [6349, 1, 1429, 3365, 1, 377, 1871, 777, 28, 10]... 길이: 64
In [7]:
import torch
from torch.utils.data import Dataset, DataLoader
class AGNewsDataset(Dataset):
def __init__(self, df, vocab, max_len):
self.ids = [text_to_ids(t, vocab, max_len) for t in df['text']]
self.labels = df['label'].tolist()
def __len__(self):
return len(self.labels)
def __getitem__(self, idx):
return (
torch.tensor(self.ids[idx], dtype=torch.long),
torch.tensor(self.labels[idx], dtype=torch.long)
)
train_dataset = AGNewsDataset(df_tr, vocab, MAX_LEN)
val_dataset = AGNewsDataset(df_val, vocab, MAX_LEN)
test_dataset = AGNewsDataset(df_test, vocab, MAX_LEN)
In [8]:
import torch
import torch.nn as nn
import torch.nn.functional as F
import math
class PositionalEncoding(nn.Module):
def __init__(self, embed_dim, max_len=512, dropout=0.1):
super().__init__()
self.dropout = nn.Dropout(dropout)
# 위치 인코딩 행렬 계산
pe = torch.zeros(max_len, embed_dim)
position = torch.arange(0, max_len).unsqueeze(1).float()
div_term = torch.exp(torch.arange(0, embed_dim, 2).float() * (-math.log(10000.0) / embed_dim))
pe[:, 0::2] = torch.sin(position * div_term) # 짝수 차원: sin
pe[:, 1::2] = torch.cos(position * div_term) # 홀수 차원: cos
pe = pe.unsqueeze(0) # [1, max_len, embed_dim]
self.register_buffer('pe', pe)
def forward(self, x):
x = x + self.pe[:, :x.size(1)]
return self.dropout(x)
class TransformerClassifier(nn.Module):
def __init__(self, vocab_size, embed_dim, num_heads, num_layers, hidden_dim, output_dim, max_len=512, dropout=0.1):
super().__init__()
# 처음부터 학습하는 임베딩
self.embedding = nn.Embedding(vocab_size, embed_dim, padding_idx=0)
self.pos_encoding = PositionalEncoding(embed_dim, max_len, dropout)
# Transformer 인코더 (소형: 2층, 2헤드)
encoder_layer = nn.TransformerEncoderLayer(
d_model=embed_dim,
nhead=num_heads,
dim_feedforward=hidden_dim,
dropout=dropout,
batch_first=True
)
self.transformer = nn.TransformerEncoder(encoder_layer, num_layers=num_layers)
self.dropout = nn.Dropout(dropout)
self.fc = nn.Linear(embed_dim, output_dim)
def forward(self, x):
# 패딩 마스크 생성
padding_mask = (x == 0)
embedded = self.pos_encoding(self.embedding(x)) # [batch, seq, embed_dim]
encoded = self.transformer(embedded, src_key_padding_mask=padding_mask)
# [CLS] 토큰 대신 평균 풀링 사용
out = encoded.mean(dim=1)
return self.fc(self.dropout(out))
model_transformer = TransformerClassifier(
vocab_size=len(vocab),
embed_dim=128, # 소형
num_heads=2, # 소형
num_layers=2, # 소형
hidden_dim=256, # 소형
output_dim=4
).to(device)
total_params = sum(p.numel() for p in model_transformer.parameters())
print(f"총 파라미터 수: {total_params:,}")
총 파라미터 수: 2,825,476
In [9]:
from torch.utils.data import Dataset, DataLoader
train_loader = DataLoader(train_dataset, batch_size=256, shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=256, shuffle=False)
test_loader = DataLoader(test_dataset, batch_size=256, shuffle=False)
print(f"Train 배치 수: {len(train_loader)}")
print(f"Val 배치 수: {len(val_loader)}")
print(f"Test 배치 수: {len(test_loader)}")
Train 배치 수: 375 Val 배치 수: 94 Test 배치 수: 30
In [10]:
from tqdm import tqdm
optimizer = torch.optim.Adam(model_transformer.parameters(), lr=1e-3)
criterion = nn.CrossEntropyLoss()
EPOCHS = 30
history = {'train_loss': [], 'train_acc': [], 'val_loss': [], 'val_acc': []}
for epoch in range(EPOCHS):
model_transformer.train()
total_loss, correct, total = 0, 0, 0
pbar = tqdm(train_loader, desc=f"Epoch {epoch+1:2d}/{EPOCHS}", leave=False)
for X_b, y_b in pbar:
X_b, y_b = X_b.to(device), y_b.to(device)
optimizer.zero_grad()
out = model_transformer(X_b) # ← out, _ 아님
loss = criterion(out, y_b)
loss.backward()
torch.nn.utils.clip_grad_norm_(model_transformer.parameters(), 1.0)
optimizer.step()
total_loss += loss.item()
correct += (out.argmax(1) == y_b).sum().item()
total += len(y_b)
pbar.set_postfix({'loss': f'{total_loss/len(pbar):.4f}', 'acc': f'{correct/total:.4f}'})
train_loss = total_loss / len(train_loader)
train_acc = correct / total
model_transformer.eval()
val_loss, val_correct, val_total = 0, 0, 0
with torch.no_grad():
for X_b, y_b in val_loader:
X_b, y_b = X_b.to(device), y_b.to(device)
out = model_transformer(X_b) # ← out, _ 아님
val_loss += criterion(out, y_b).item()
val_correct += (out.argmax(1) == y_b).sum().item()
val_total += len(y_b)
val_loss = val_loss / len(val_loader)
val_acc = val_correct / val_total
history['train_loss'].append(train_loss)
history['train_acc'].append(train_acc)
history['val_loss'].append(val_loss)
history['val_acc'].append(val_acc)
print(f"\rEpoch {epoch+1:2d}/{EPOCHS} | Train Loss: {train_loss:.4f} | Train Acc: {train_acc:.4f} | Val Loss: {val_loss:.4f} | Val Acc: {val_acc:.4f}", end='', flush=True)
print()
Epoch 1/30 | Train Loss: 0.7005 | Train Acc: 0.7205 | Val Loss: 0.5081 | Val Acc: 0.8443
Epoch 2/30 | Train Loss: 0.3877 | Train Acc: 0.8624 | Val Loss: 0.4145 | Val Acc: 0.8818
Epoch 3/30 | Train Loss: 0.3082 | Train Acc: 0.8910 | Val Loss: 0.3993 | Val Acc: 0.8781
Epoch 4/30 | Train Loss: 0.2618 | Train Acc: 0.9088 | Val Loss: 0.3513 | Val Acc: 0.8970
Epoch 5/30 | Train Loss: 0.2310 | Train Acc: 0.9195 | Val Loss: 0.3329 | Val Acc: 0.8992
Epoch 6/30 | Train Loss: 0.2059 | Train Acc: 0.9284 | Val Loss: 0.3273 | Val Acc: 0.8992
Epoch 7/30 | Train Loss: 0.1863 | Train Acc: 0.9344 | Val Loss: 0.3025 | Val Acc: 0.9032
Epoch 8/30 | Train Loss: 0.1699 | Train Acc: 0.9402 | Val Loss: 0.2978 | Val Acc: 0.9078
Epoch 9/30 | Train Loss: 0.1557 | Train Acc: 0.9454 | Val Loss: 0.2954 | Val Acc: 0.9067
Epoch 10/30 | Train Loss: 0.1392 | Train Acc: 0.9507 | Val Loss: 0.2890 | Val Acc: 0.9041
Epoch 11/30 | Train Loss: 0.1308 | Train Acc: 0.9531 | Val Loss: 0.2778 | Val Acc: 0.9093
Epoch 12/30 | Train Loss: 0.1211 | Train Acc: 0.9571 | Val Loss: 0.2776 | Val Acc: 0.9088
Epoch 13/30 | Train Loss: 0.1118 | Train Acc: 0.9599 | Val Loss: 0.2844 | Val Acc: 0.9072
Epoch 14/30 | Train Loss: 0.1052 | Train Acc: 0.9620 | Val Loss: 0.3016 | Val Acc: 0.8990
Epoch 15/30 | Train Loss: 0.0966 | Train Acc: 0.9655 | Val Loss: 0.2814 | Val Acc: 0.9081
Epoch 16/30 | Train Loss: 0.0899 | Train Acc: 0.9673 | Val Loss: 0.2762 | Val Acc: 0.9087
Epoch 17/30 | Train Loss: 0.0856 | Train Acc: 0.9691 | Val Loss: 0.2819 | Val Acc: 0.9099
Epoch 18/30 | Train Loss: 0.0796 | Train Acc: 0.9709 | Val Loss: 0.2727 | Val Acc: 0.9135
Epoch 19/30 | Train Loss: 0.0764 | Train Acc: 0.9724 | Val Loss: 0.2699 | Val Acc: 0.9114
Epoch 20/30 | Train Loss: 0.0696 | Train Acc: 0.9746 | Val Loss: 0.2796 | Val Acc: 0.9110
Epoch 21/30 | Train Loss: 0.0678 | Train Acc: 0.9753 | Val Loss: 0.2985 | Val Acc: 0.9073
Epoch 22/30 | Train Loss: 0.0627 | Train Acc: 0.9768 | Val Loss: 0.2935 | Val Acc: 0.9089
Epoch 23/30 | Train Loss: 0.0598 | Train Acc: 0.9780 | Val Loss: 0.2813 | Val Acc: 0.9119
Epoch 24/30 | Train Loss: 0.0577 | Train Acc: 0.9795 | Val Loss: 0.2885 | Val Acc: 0.9094
Epoch 25/30 | Train Loss: 0.0530 | Train Acc: 0.9804 | Val Loss: 0.2909 | Val Acc: 0.9109
Epoch 26/30 | Train Loss: 0.0528 | Train Acc: 0.9810 | Val Loss: 0.2945 | Val Acc: 0.9070
Epoch 27/30 | Train Loss: 0.0494 | Train Acc: 0.9821 | Val Loss: 0.2973 | Val Acc: 0.9085
Epoch 28/30 | Train Loss: 0.0467 | Train Acc: 0.9828 | Val Loss: 0.3021 | Val Acc: 0.9087
Epoch 29/30 | Train Loss: 0.0479 | Train Acc: 0.9822 | Val Loss: 0.3002 | Val Acc: 0.9116
Epoch 30/30 | Train Loss: 0.0428 | Train Acc: 0.9844 | Val Loss: 0.3110 | Val Acc: 0.9065
In [11]:
import matplotlib.pyplot as plt
fig, axes = plt.subplots(1, 2, figsize=(14, 5))
epochs = range(1, EPOCHS + 1)
axes[0].plot(epochs, history['train_loss'], 'b-o', markersize=5, label='Train Loss')
axes[0].plot(epochs, history['val_loss'], 'r-o', markersize=5, label='Val Loss')
axes[0].set_title('Loss 곡선', fontsize=13, fontweight='bold')
axes[0].set_xlabel('Epoch')
axes[0].set_ylabel('Loss')
axes[0].legend()
axes[0].grid(alpha=0.3)
axes[1].plot(epochs, history['train_acc'], 'b-o', markersize=5, label='Train Acc')
axes[1].plot(epochs, history['val_acc'], 'r-o', markersize=5, label='Val Acc')
axes[1].set_title('Accuracy 곡선', fontsize=13, fontweight='bold')
axes[1].set_xlabel('Epoch')
axes[1].set_ylabel('Accuracy')
axes[1].set_ylim(0, 1)
axes[1].legend()
axes[1].grid(alpha=0.3)
plt.suptitle(f'Transformer (학습가능 임베딩) 학습 결과 | 최고 Val Acc: {max(history["val_acc"]):.4f}',
fontsize=14, fontweight='bold')
plt.tight_layout()
plt.show()
In [12]:
import seaborn as sns
import numpy as np
from sklearn.metrics import confusion_matrix, classification_report
label_names = ['World', 'Sports', 'Business', 'Sci/Tech']
model_transformer.eval()
all_preds = []
with torch.no_grad():
for X_b, y_b in test_loader:
X_b = X_b.to(device)
out = model_transformer(X_b)
all_preds.extend(out.argmax(1).cpu().numpy())
all_preds = np.array(all_preds)
y_test_arr = df_test['label'].values
test_acc = (all_preds == y_test_arr).mean()
print(f"✅ 테스트 정확도: {test_acc:.4f} ({test_acc*100:.2f}%)")
print()
print(classification_report(y_test_arr, all_preds, target_names=label_names))
cm = confusion_matrix(y_test_arr, all_preds)
fig, axes = plt.subplots(1, 2, figsize=(14, 5))
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues',
xticklabels=label_names, yticklabels=label_names, ax=axes[0])
axes[0].set_title('혼동행렬 (개수)', fontsize=13, fontweight='bold')
axes[0].set_ylabel('실제 레이블')
axes[0].set_xlabel('예측 레이블')
cm_pct = cm.astype(float) / cm.sum(axis=1, keepdims=True)
sns.heatmap(cm_pct, annot=True, fmt='.2%', cmap='Blues',
xticklabels=label_names, yticklabels=label_names, ax=axes[1])
axes[1].set_title('혼동행렬 (비율)', fontsize=13, fontweight='bold')
axes[1].set_ylabel('실제 레이블')
axes[1].set_xlabel('예측 레이블')
plt.suptitle(f'Transformer (학습가능 임베딩) 테스트 결과 | 정확도: {test_acc*100:.2f}%',
fontsize=14, fontweight='bold')
plt.tight_layout()
plt.show()
✅ 테스트 정확도: 0.9064 (90.64%)
precision recall f1-score support
World 0.89 0.93 0.91 1900
Sports 0.96 0.96 0.96 1900
Business 0.87 0.87 0.87 1900
Sci/Tech 0.90 0.86 0.88 1900
accuracy 0.91 7600
macro avg 0.91 0.91 0.91 7600
weighted avg 0.91 0.91 0.91 7600
