Thanks for your reply. It works now

Here's the source:

[pascal]
//StartMousePos is the starting point of the mouse in screen coordinates
//MousePos is the current position of the mouse in screen coordinates
//UVPos is the current UV-position of the viewport center
//StartUVPos is the initial UV-position of the viewport center (when we started dragging).

procedure TMainForm.ViewUVMouseDown(Sender: TObject; Button: TMouseButton;
Shift: TShiftState; X, Y: Integer);
begin
if Button = mbRight then
begin
DragMoving := True;
StartMousePos.X := X;
StartMousePos.Y := Y;
StartUVPos.u := UVPos.u;
StartUVPos.v := UVPos.v;
end;
end;

procedure TMainForm.ViewUVMouseMove(Sender: TObject; Shift: TShiftState; X, Y: Integer);
var
uFactor, vFactor: Single;
begin
if DragMoving then
begin
MousePos.X := X;
MousePos.Y := Y;

//Scales displacement-vector from screen to UV-coordinates (Flip Y-axis)
uFactor := UVSize/ViewUV.Width;
vFactor := -(UVSize/ViewUV.Height);

//Transform displacement vector and combine it with the initial viewport position
UVPos.u := StartUVPos.u - (MousePos.x - StartMousePos.x)*uFactor;
UVPos.v := StartUVPos.v - (MousePos.y - StartMousePos.y)*vFactor;
end;
end;

procedure TMainForm.ViewUVMouvseUp(Sender: TObject; Button: TMouseButton;
Shift: TShiftState; X, Y: Integer);
begin
if Button = mbRight then
begin
DragMoving := False;
MousePos.X := 0;
MousePos.Y := 0;
StartMousePos.X := 0;
StartMousePos.Y := 0;
end;
end;
[/pascal]

Instead of using gluUnProject, i figured that it was easier to just transform by hand. If you want to rotate your view, this can be tedious though, but i don't need this. I got rid of the "feedback" effect now.