#C, 516

Linefeeds added for improved legibility presentation. (Legibility went out the window, I'm afraid.)

#define Z(u,v) putchar(o[u]),putchar(o[v])
#define X while((Y=getchar())>31){Y&=223;if(Y==74)Y--;if(Y<65||Y>90
P,L,A,Y,f,a,i,r,c=512,o[25],d[2],*e=o;Q(){for(i=0;o[i]!=d[0];i++);i-=(f=i%5);
for(r=0;o[r]!=d[1];r++);r-=(a=r%5);if(f==a)Z(f+(i+5)%25,a+(r+5)%25);
else if(i==r)Z((f+1)%5+i,(a+1)%5+r);else Z(a+i,f+r);}main(){X||c&(A=1<<Y-65))continue;
c|=A;*e++=Y;}A=1;Y=65;for(P=0;P<25;P++){if(!(c&A))*e++=Y;
if(++Y==74)Y++,A+=A;A+=A;}L=0;X)continue;if(L&&Y==*d)d[1]=88,Q(),*d=Y;
else d[L]=Y,L=1-L;if(!L)Q();}if(L)d[1]=88,Q();}

Example:

$ ./pf
Playfair                                    
The quick brown fox jumps over the lazy dog
QMHNPEKSCBQVTPSVEPEFTQUGDOKGAYXFRTKV

C, 516

Linefeeds added for improved legibility presentation. (Legibility went out the window, I'm afraid.)

#define Z(u,v) putchar(o[u]),putchar(o[v])
#define X while((Y=getchar())>31){Y&=223;if(Y==74)Y--;if(Y<65||Y>90
P,L,A,Y,f,a,i,r,c=512,o[25],d[2],*e=o;Q(){for(i=0;o[i]!=d[0];i++);i-=(f=i%5);
for(r=0;o[r]!=d[1];r++);r-=(a=r%5);if(f==a)Z(f+(i+5)%25,a+(r+5)%25);
else if(i==r)Z((f+1)%5+i,(a+1)%5+r);else Z(a+i,f+r);}main(){X||c&(A=1<<Y-65))continue;
c|=A;*e++=Y;}A=1;Y=65;for(P=0;P<25;P++){if(!(c&A))*e++=Y;
if(++Y==74)Y++,A+=A;A+=A;}L=0;X)continue;if(L&&Y==*d)d[1]=88,Q(),*d=Y;
else d[L]=Y,L=1-L;if(!L)Q();}if(L)d[1]=88,Q();}

Example:

$ ./pf
Playfair                                    
The quick brown fox jumps over the lazy dog
QMHNPEKSCBQVTPSVEPEFTQUGDOKGAYXFRTKV