\(\left(x^2-3x\right)^2-14x^2+42x+40=\left[\left(x^2-3x\right)^2-14\left(x^2-3x\right)+49\right]-9=\left(x^2-3x-7\right)-3^3=\left(x^2-3x-7-3\right)\left(x^2-3x-7+3\right)=\left(x^2-3x-10\right)\left(x^2-3x-4\right)=\left(x-5\right)\left(x+2\right)\left(x-4\right)\left(x+1\right)\)
\(\left(x^2-3x\right)^2-14x^2+42x+40\\ =x^4-6x^3+9x^2-14x^2+42x+40\\ =x^4-6x^3-5x^2+42x+40\\ =x^4+x^3-7x^3-7x^2+2x^2+2x+40x+40\\ =\left(x+1\right)\left(x^3-7x^2+2x+40\right)\\ =\left(x+1\right)\left(x^3+2x^2-9x^2-18x+20x+40\right)\\ =\left(x+1\right)\left(x+2\right)\left(x^2-9x+20\right)\\ =\left(x-5\right)\left(x-4\right)\left(x+1\right)\left(x+2\right)\)
\(\left(x^2-3x\right)^2-14x^2+42x+40\)
\(=\left(x^2-3x\right)^2-14\left(x^2-3x\right)+40\)
\(=\left(x^2-3x-4\right)\left(x^2-3x-10\right)\)
\(=\left(x-4\right)\left(x+1\right)\left(x-5\right)\left(x+2\right)\)