Why is my picking function behaving unexpectedly?

I have been trying to get this thing to work for 2 hours but no luck.

BOOL Directx::Picking(HWND hWnd, AnimatedMesh *entity)
{
    D3DXMATRIX matProj;
    POINT pt;
    D3DVIEWPORT9 vp;
    D3DXMATRIX matWorld;
    D3DXMATRIX matView;

    GetCursorPos(&pt);
    ScreenToClient(hWnd, &pt);
    d3ddev->GetTransform(D3DTS_PROJECTION, &matProj);
    d3ddev->GetViewport(&vp);
    d3ddev->GetTransform(D3DTS_VIEW, &matView);

    d3ddev->GetTransform(D3DTS_WORLD, &matWorld);
    // Use inverse of matrix
    D3DXVECTOR3 rayPos((float)pt.x, (float)pt.y,0); // near-plane position
    D3DXVECTOR3 rayDir((float)pt.x, (float)pt.y,1); // far-plane position  
    D3DXVec3Unproject(&rayPos,&rayPos,&vp,&matProj,&matView,&matWorld);
    D3DXVec3Unproject(&rayDir,&rayDir,&vp,&matProj,&matView,&matWorld);
    rayDir -= rayPos; // make a direction from the 2 positions
    D3DXVec3Normalize(&rayDir,&rayDir);
    float distanceToCollision;
    BOOL hasHit = 0;
    if(FAILED(D3DXIntersect(entity->pDrawMesh, &rayPos, &rayDir, &hasHit, NULL, NULL, NULL, &distanceToCollision, NULL, NULL)))
    {
        PostQuitMessage(0);
    };

    if(hasHit!=0)
    {
        PostQuitMessage(0);
    }
    return hasHit;
}

I think the problem with the matWorld because I'm getting the world transformation instead of object transformation but that is just guess. I found out new thing, I'm loading 2 animated meshes, first one a cube and second is just plane. I have no problem with rendering them, but when i test picking against the cube, it will say that it was picked even when it wasn't. but for the plane, when you click on it says it h开发者_如何学JAVAaven' t been clicked.

EDIT

Okay so i did more debugging and i got some result about the problem. So, I tried to draw my ray! When i draw my ray it is suppose to draw a dot where i clicked. But when I draw my ray the point got drawed close to the origin(0,0,0) i clicked again than it moved down a little and it kept going up and down.

Here is how i draw my ray

............
  D3DXVec3Normalize(&rayDir,&rayDir);
    Program::mesh1->FillInTransformation(0, 0, 0, 0, 0, 0, rayPos.x, rayPos.y, Program::z, true, false, false);  //HERE I TOOK X AND Y OF rayPos AND DRAW DOT ON IT'S POSITION
    float distanceToCollision;
    BOOL hasHit = 0;
    if(FAILED(D3DXIntersect(entity->pDrawMesh, &rayPos, &rayDir, &hasHit, NULL, NULL, NULL, &distanceToCollision, NULL, NULL)))
    {
        PostQuitMessage(0);
    };
........

---

Well try this:

BOOL Directx::Picking(HWND hWnd, AnimatedMesh *entity)
{
    POINT pt;
    D3DVIEWPORT9 vp;
    D3DXMATRIX matProj, matView;

    GetCursorPos(&pt);
    ScreenToClient(hWnd, &pt);

    float w = (float)backBufferWidth;
    float h = (float)backBufferHeight;

    d3ddev->GetTransform(D3DTS_VIEW, &matView);
    d3ddev->GetTransform(D3DTS_PROJECTION, &matProj);
    d3ddev->GetViewport(&vp);

    //Transform cursor position to view space
    float x = (2.0f*pt.x/w - 1.0f) / matProj(0,0);
    float y = (-2.0f*pt.y/h + 1.0f) / matProj(1,1);


    D3DXVECTOR3 rayOrigin(0.0f, 0.0f,0.0f); // near-plane position
    D3DXVECTOR3 rayDir(x, y, 1.0f); // far-plane position

    D3DXMATRIX matInvView;

    D3DXMatrixInverse(&matInvView, 0, &matView);

    D3DXVECTOR3 rayOriginW, rayDirW;

    // Transform picking ray to world space.
    D3DXVec3TransformCoord(&rayOriginW, &rayOrigin, &invView);
    D3DXVec3TransformNormal(&rayDirW, &rayDdir, &invView);
    D3DXVec3Normalize(&rayDirW, &rayDirW);

    BOOL hasHit;
    float distanceToCollision;

    if(FAILED(D3DXIntersect(entity->pDrawMesh, &rayOriginW, &rayDirW, &hasHit, NULL, NULL, NULL, &distanceToCollision, NULL, NULL)))
    {
        PostQuitMessage(0);
    };

    s=rayPos;

    return hasHit;
}