a: \(\left(ab-1\right)^2+\left(a+b\right)^2\)
\(=a^2b^2-2ab+1+a^2+2ab+b^2\)
\(=a^2b^2+a^2+b^2+1\)
\(=a^2\left(b^2+1\right)+\left(b^2+1\right)=\left(a^2+1\right)\left(b^2+1\right)\)
b: \(\left(a^2+4b^2-5\right)^2-16\left(ab+1\right)^2\)
\(=\left(a^2+4b^2-5\right)^2-\left(4ab+4\right)^2\)
\(=\left(a^2+4b^2-5-4ab-4\right)\left(a^2+4b^2-5+4ab+4\right)\)
\(=\left\lbrack\left(a-2b\right)^2-9\right\rbrack\cdot\left\lbrack\left(a+2b\right)^2-1\right\rbrack\)
=(a-2b-3)(a-2b+3)(a+2b-1)(a+2b+1)
c: \(4a^2b^2-\left(a^2+b^2-c^2\right)^2\)
\(=\left(2ab\right)^2-\left(a^2+b^2-c^2\right)^2\)
\(=\left(2ab-a^2-b^2+c^2\right)\left(2ab+a^2+b^2-c^2\right)\)
\(=\left\lbrack c^2-\left(a^2-2ab+b^2\right)\right\rbrack\left\lbrack\left(a^2+2ab+b^2\right)-c^2\right\rbrack\)
\(=\left\lbrack c^2-\left(a-b\right)^2\right\rbrack\left\lbrack\left(a+b\right)^2-c^2\right\rbrack\)
=(c-a+b)(c+a-b)(a+b-c)(a+b+c)
d: \(x^3+y^3+z^3-3xyz\)
\(=\left(x+y\right)^3-3xy\left(x+y\right)+z^3-3xyz\)
\(=\left(x+y\right)^3+z^3-3xy\left(x+y+z\right)\)
\(=\left(x+y+z\right)\left\lbrack\left(x+y\right)^2-\left(x+y\right)\cdot z+z^2\right\rbrack-3xy\left(x+y+z\right)\)
\(=\left(x+y+z\right)\left(x^2+2xy+y^2-xz-yz+z^2-3xy\right)\)
\(=\left(x+y+z\right)\left(x^2+y^2+z^2-xy-xz-yz\right)\)
e: \(a\left(a+2b\right)^3-b\left(2a+b\right)^3\)
\(=a\left(a^3+6a^2b+12ab^2+8b^3\right)-b\left(8a^3+12a^2b+6ab^2+b^3\right)\)
\(=a^4+6a^3b+12a^2b^2+8ab^3-8a^3b-12a^2b^2-6ab^3-b^4\)
\(=a^4-2a^3b+2ab^3-b^4=\left(a^4-b^4\right)-2ab\left(a^2-b^2\right)\)
\(=\left(a^2-b^2\right)\left(a^2+b^2-2ab\right)=\left(a-b\right)\left(a+b\right)\cdot\left(a-b\right)^2=\left(a+b\right)\cdot\left(a-b\right)^3\)
f: \(\left(x+y+z\right)^3-x^3-y^3-z^3\)
\(=\left(x+y+z-x\right)\left\lbrack\left(x+y+z\right)^2+x\left(x+y+z\right)+x^2\right\rbrack-\left(y+z\right)\left(y^2-yz+z^2\right)\)
\(=\left(y+z\right)\left\lbrack x^2+y^2+z^2+2xy+2yz+2xz+x^2+xy+xz+x^2\right\rbrack-\left(y+z\right)\left(y^2-yz+z^2\right)\)
\(=\left(y+z\right)\left(3x^2+3xy+y^2+z^2+3xz+2yz-y^2+yz-z^2\right)\)
\(=\left(y+z\right)\left(3x^2+3xy+3xz+3yz\right)\)
=3(y+z)\(\left(x^2+xy+xz+yz\right)\)
=3(y+z)[x(x+y)+z(x+y)]
=3(y+z)(x+y)(x+z)

