mechanisms.py · 可读全文

这里保留原文的中文、公式标记、代码和链接。可直接交给 Agent 读取，也可选取下方原文复制。此入口返回 HTML；原始 .md / .py 文件仍保留，完整候选包同时包含 UTF-8 原件。

"""Small explicit mechanisms and plotting helpers. No claim of learned semantics."""
from pathlib import Path
import re, itertools, copy
import numpy as np
import matplotlib.pyplot as plt
from PIL import Image
import legacy_numpy as old

def counts_svd(window=3,dimensions=4):
    sentences=old.route_a()['sentences'];docs=[re.findall('[a-z]+',s.lower()) for s in sentences]
    vocab=sorted(set(itertools.chain.from_iterable(docs)));ids={w:i for i,w in enumerate(vocab)}
    counts=np.zeros((len(vocab),len(vocab)))
    for doc in docs:
        for i,w in enumerate(doc):
            for j in range(max(0,i-window),min(len(doc),i+window+1)):
                if i!=j:counts[ids[w],ids[doc[j]]]+=1
    denom=counts.sum(1)[:,None]*counts.sum(0)[None,:]
    ratio=np.divide(counts*counts.sum(),denom,out=np.zeros_like(counts),where=denom>0)
    ppmi=np.maximum(np.log(np.maximum(ratio,1e-300)),0)
    u,s,_=np.linalg.svd(ppmi,full_matrices=False);emb=u[:,:dimensions]*np.sqrt(s[:dimensions])
    return vocab,counts,ppmi,emb,docs

def neighbors(vocab,emb,word='bank',n=6):
    i=vocab.index(word);norms=np.linalg.norm(emb,axis=1)
    sim=np.divide(emb@emb[i],norms*norms[i],out=np.zeros(len(vocab)),where=norms*norms[i]>1e-12)
    order=[j for j in np.argsort(-sim) if j!=i][:n]
    return [(vocab[j],float(sim[j]),bool(norms[j]<1e-12)) for j in order]

def block_parameters():
    b=old.classroom_parameters()['BLOCK']
    p={k:np.array(v,float) for k,v in {'Wq':np.concatenate([b['WQ1'],b['WQ2']],1),
       'Wk':np.concatenate([b['WK1'],b['WK2']],1),'Wv':np.concatenate([b['WV1'],b['WV2']],1),
       'Wo':b['WO'],'W1':b['W1'],'W2':b['W2'],'b1':np.zeros(6),'b2':np.zeros(4),
       'g1':np.ones(4),'g2':np.ones(4),'bN1':np.zeros(4),'bN2':np.zeros(4)}.items()}
    return np.array(b['X'],float),p

def block_forward(x,p,pre_norm=False):
    norm=lambda z,i:old.layer_norm(z,p['g'+str(i)],p['bN'+str(i)])
    att,tr=old.multihead(norm(x,1) if pre_norm else x,p)
    residual=x+att;u=residual if pre_norm else norm(residual,1)
    ff_input=norm(u,2) if pre_norm else u
    mid=np.maximum(ff_input@p['W1']+p['b1'],0);ff=mid@p['W2']+p['b2']
    out=u+ff if pre_norm else norm(u+ff,2)
    return out,dict(input=x,**tr,attention=att,residual1=residual,norm1=u,ff_hidden=mid,ff=ff,output=out)

def vision_parameters(seed=20261016):
    rng=np.random.default_rng(seed)
    # Human-readable RGB patch means + x/y coordinates, then untrained projections.
    return {k:rng.normal(0,.3,shape) for k,shape in {'encoder':(5,8),'K':(8,4),'V':(8,4),
        'Q':(8,4),'O':(4,8),'bias':(8,),'head':(8,6),'embedding':(6,8)}.items()}

def vision_read(image_path,p,prefix=(0,1),enabled=True):
    image=np.asarray(Image.open(image_path).convert('RGB').resize((224,224)),float)/255
    patches=image.reshape(14,16,14,16,3).transpose(0,2,1,3,4)
    rgb=patches.mean((2,3)).reshape(196,3)
    yy,xx=np.meshgrid(np.linspace(0,1,14),np.linspace(0,1,14),indexing='ij')
    features=np.column_stack([rgb,xx.ravel(),yy.ravel()]);z=features@p['encoder']
    language=p['embedding'][list(prefix)].mean(0);q=language[None]@p['Q'];k=z@p['K'];v=z@p['V']
    read,weights=old.attention(q,k,v)
    projected=read[0]@p['O']+p['bias']
    branch=float(enabled)*projected  # gate AFTER bias, BEFORE language residual
    logits=(language+branch)@p['head'];probabilities=old.softmax(logits)
    return dict(image=image,features=features,Z=z,Q=q,K=k,V=v,weights=weights,
                read=read,branch=branch,logits=logits,probabilities=probabilities)

def heatmap(a,xlabels,ylabels,title,unit='weight'):
    fig,ax=plt.subplots(figsize=(max(5,len(xlabels)*.35),max(2.3,len(ylabels)*.3)))
    im=ax.imshow(a,aspect='auto',cmap='YlGnBu');ax.set_xticks(range(len(xlabels)),xlabels,rotation=60,ha='right')
    ax.set_yticks(range(len(ylabels)),ylabels);ax.set_title(title);fig.colorbar(im,ax=ax,label=unit)
    fig.tight_layout();return fig

OLD_TREE={():{'A':.6,'B':.4},('A',):{'X':.51,'Y':.49},('B',):{'X':.9,'Y':.1},
    ('A','X'):{'<EOS>':1.},('A','Y'):{'<EOS>':1.},('B','X'):{'<EOS>':1.},('B','Y'):{'<EOS>':1.}}
CLASSROOM_TREE={():{'A':.45,'B':.30,'C':.25},('A',):{'a':.7,'b':.3},
    ('A','a'):{'<EOS>':2/3,'c':1/3},('A','a','c'):{'<EOS>':1.},('A','b'):{'<EOS>':1.},
    ('B',):{'<EOS>':.75,'d':.25},('B','d'):{'<EOS>':1.},('C',):{'<EOS>':1.}}

def tree_figure(tree):
    leaves=old.exhaustive_paths(tree);positions={tuple(r['tokens']):(i,-len(r['tokens'])) for i,r in enumerate(leaves)}
    def locate(prefix):
        if prefix in positions:return positions[prefix]
        children=[locate(prefix+(t,)) for t in tree[prefix]]
        positions[prefix]=(np.mean([p[0] for p in children]),-len(prefix));return positions[prefix]
    locate(());fig,ax=plt.subplots(figsize=(8,4))
    for prefix,probs in tree.items():
        x,y=positions[prefix]
        for tok,p in probs.items():
            nx,ny=positions[prefix+(tok,)];ax.plot([x,nx],[y,ny],color='#6f91ab')
            ax.text((x+nx)/2,(y+ny)/2,str(round(p,4)),fontsize=9)
    for prefix,(x,y) in positions.items():ax.text(x,y,prefix[-1] if prefix else 'BOS',ha='center',bbox=dict(fc='white',ec='#003262',boxstyle='round'))
    ax.set_title('Constructed tree: normalized edge probabilities');ax.axis('off');fig.tight_layout();return fig