I have adapted the solution from the link provided in the comment.
Finally I have changed my mind. Instead of using a trilist matching the standard structure and trying to adapt my vertex order to this trilist, I use this method sorting vertex clockwise and then I generated a trilist in a more general way based on this clockwise ordering.
Below a result that works for me although I did not check with culling as I don't want to cull.
Using DX11 and XMVECTOR type with no SSE intrinsic.
PDPATOM is a struct describing an atom with position XYZ Pos and a FLOAT Angle
LPPDBATOM is a pointer to a PDPATOM struct
LPPDBATOM* is a list of LPPDBATOM
ACount is a DWORD for the number of LPPDBATOM in the pointer list
//sorting function based on Angle value
int __cdecl SortAtomDistance(const VOID* arg1, const VOID* arg2)
{
LPPDBATOM a1 = *(LPPDBATOM*)arg1;
LPPDBATOM a2 = *(LPPDBATOM*)arg2;
if (a1-> Angle > a2-> Angle) return +1;
if (a1-> Angle < a2-> Angle) return -1;
return 0;
}
//function that create a temporary copy of the original list to work with
//use qsort to order the atom based on the Angle value
//at the end copy the temporary list to the original one
void SortAtomPoly(LPPDBATOM* ppAList, DWORD ACount)
{
LPPDBATOM* ppAListTmp = new LPPDBATOM[ACount];
memcpy(ppAListTmp, ppAList, sizeof(LPPDBATOM)*ACount);
qsort(ppAListTmp, ACount, sizeof(LPPDBATOM), SortAtomDistance);
memcpy(ppAList, ppAListTmp, sizeof(LPPDBATOM)*ACount);
delete[] ppAListTmp;
}
//main function that generate a planar geometry for my needs.
//note that the geometry is expected to be flat convex and to be a tri,
//quad, pent or hex or more but here limited to a hex.
//parameters function are ppAList : the original list of atom, ACount
//the number, Faces the number of poly to be generated (e.g. Face = 1
//means tri, 2=quad, etc…
void CGeometryDX::SetPolygon(LPPDBATOM* ppAList, DWORD ACount, DWORD Faces)
{
//calculate the center from the positions
XMVECTOR Centroid = XMVectorSet(0, 0, 0, 0);
for (DWORD a = 0; a < ACount; a++)
{
LPPDBATOM pA = ppAList[a];
Centroid += pA->Pos;
}
Centroid /= ACount;
//Get a plane equation from the three first positions. We expect a planar geometry
XMVECTOR Plane = XMPlaneFromPoints(ppAList[0]->Pos, ppAList[1]->Pos, ppAList[2]->Pos);
// Inverse the Plane,
XMVECTOR QuadXY = XMQuaternionInverse(Plane);
//for each position calculate the delta to centroid and multiply by
//inverse plane. Use the result XY to get the Angle representing the
//orientation (clockwise) of the position from the centroid (radian)
for (DWORD a = 0; a < ACount; a++)
{
LPPDBATOM pA = ppAList[a];
XMVECTOR XY = QuadXY*(pA->Pos-Centroid);
pA->Angle = atan2f(XY.y, XY.x);
}
//then sort
SortAtomPoly(ppAList, ACount);
Vertex V[6];//we want vertex list for hexagon shape : max 6 vertices
for (DWORD a = 0; a < ACount; a++)
{
LPPDBATOM pA = ppAList[a];
XMStoreFloat3(&V[a].Pos, pA->Pos);
V[a].Color = XMVectorSet(1.0, 1.0, 1.0, 0.5);//transparent white
V[a].Tex = XMFLOAT2(0, 0);//no texture applied
V[a].Normal = XMFLOAT3(0, 1, 0);//default normal
}
//for this application we will have only convex geometry so
//generate a list of tris in a trianglefan mode
//(old DX7/8 trilist type)
//with point zero as starting vertex for all tris.
//alternative generation of indices is possible
WORD IBs[12];//we want tri list for hexagon shape : max 4*3 indices
//loop based on Faces param, 1 = tri, 2 = quad, etc...
//should work for more than hexagon
DWORD i = 0;//incremented each loop step to turn around point zero
//with increasing vertex index. E.g. 0,1,2 -> 0,2,3…etc
for (DWORD f = 0; f < Faces; f++)
{
IBs[f * 3 + 0] = 0;
IBs[f * 3 + 1] = i + 1;
IBs[f * 3 + 2] = i + 2;
i ++;
}
CreatMesh(V, Faces+2, IBs, Faces*3, D3D11_PRIMITIVE_TOPOLOGY_TRIANGLELIST);
}