I/O Completion Ports mit Async Sockets

[FBIagent]

Here i go

Ich habe ein kleines Problem mit I/O Completion Ports und Async Sockets.
Da ich noch nicht der Erfahrenste auf dem Gebiet bin, habe ich mir als vorlage ein Projekt
von CodeProject herunter geladen(Darf ich die Seite hier verlinken? Weis ich grad nicht).
So weit so gut: Ein klein wenig gelern um erstmal die Ansätze zu verstehen und herum experimentiert. Nun habe ich mir soweit Klassen zusammen gestellt, und nach meinen
Vorstellungen formatiert das ich es besser verstehe. Zum test habe ich mir jetzt einen
kleinen Echo Server mit der zusammgestellten IOCPServer Klasse geschrieben.

Den dazugehörigen echo Klienten habe ich mir irgendwo gezogen(in dem Momment war
ich zu faul selber kurz einen zu schreiben ^^)

Weitere informationen:
IOCPServer Klasse benutzt eine Klasse namens IOCPClientContext um die
connections zu händeln.

Bis jetzt habe ich noch alle bisherigen Fehler selbst ausmerzen können.
Nur jetzt zu den beiden mit denen ich nicht klar komme:
Im d'tor von IOCPClientContext betrete ich eine while schleife:

while (!HasOverlappedIoCompleted(_overlapped))

in dieser hängt der server dann (und somit auch der worker thread der gebraucht wird
um weitere Klienten zu händeln).

Desweiteren bekomme ich einen WSARecv error wenn ein Klient die verbindung trennt.
Ist für mich soweit nachvollziebar da der Klient ja nicht mehr verbunden ist, nur sollte
ich keinen Error bei WSARecv bekommen sondern eine client gone message im
worker Thread womit auch die scheife mit !HasOverlappedIOCompleted(_overlapped)
beendet werden sollte da dann auch IO completed ist.

Ouch... viel source ließtsich das überhaupt jemand durch?^^

ServerIocp.h

#ifndef _SERVER_IOCP_H_
#define _SERVER_IOCP_H_

//#define WIN32_LEAN_AND_MEAN

#include <vector>
#include <winsock2.h>
#include <windows.h>

#define WORKER_THREADS_PER_PROCESSOR 2
#define WAIT_TIMEOUT_INTERVAL 100

enum IOCP_OPC
{
    OP_NONE = 0,
    OP_READ,
    OP_WRITE
};

enum IOCP_CRR
{
    IOCP_CRR_CREATE_ERROR = 1, // error accosiate accepted socket with iocp
    IOCP_CRR_GONE, // client connection is gone
    IOCP_CRR_RECV_ERROR, // error on WSARecv
    IOCP_CRR_SEND_ERROR // error on WSASend
};

class IOCPClientContext
{
public:
    IOCPClientContext(SOCKET socket, HANDLE hIOCP)
    : _socket(socket), _overlapped(new OVERLAPPED), _hIOCP(hIOCP), _writeBufQueue(), _totalBytesTransfered(0), _totalBytesSent(0), _customContext(NULL)
    {
        ZeroMemory(_overlapped, sizeof(OVERLAPPED));
        InitializeCriticalSection(&_csOpCode);
        InitializeCriticalSection(&_csWriteBufQueue);
    }

    virtual ~IOCPClientContext()
    {
        std::cout << "IOCPClientContext::~IOCPClientContext() -> enter" << std::endl;

        while (!HasOverlappedIoCompleted(_overlapped))
          Sleep(1);

        closesocket(_socket);
        delete _overlapped;
        DeleteCriticalSection(&_csOpCode);
        DeleteCriticalSection(&_csWriteBufQueue);
        std::cout << "IOCPClientContext::~IOCPClientContext() -> leave" << std::endl;
    }

    void addToWriteBufQueue(char *buf, unsigned long len)
    {
        std::cout << "IOCPClientContext::addToWriteBufQueue() -> enter" << std::endl;
        EnterCriticalSection(&_csWriteBufQueue);
        _writeBufQueue.push_back(buf);
        _writeBufLengthQueue.push_back(len);
        LeaveCriticalSection(&_csWriteBufQueue);
        std::cout << "IOCPClientContext::addToWriteBufQueue() -> leave" << std::endl;
    }

    void setOpCode(IOCP_OPC opCode)
    {
        std::cout << "IOCPClientContext::setOpCode() -> enter" << std::endl;
        EnterCriticalSection(&_csOpCode);
        _opCode = opCode;
        LeaveCriticalSection(&_csOpCode);
        std::cout << "IOCPClientContext::setOpCode() -> leave" << std::endl;
    }

    void setTotalBytesTransfered(int totalBytesTransfered)
    {
        _totalBytesTransfered = totalBytesTransfered;
    }

    void setTotalBytesSent(int totalBytesSent)
    {
        _totalBytesSent = totalBytesSent;
    }

    void setCustomContext(void *customContext)
    {
        _customContext = customContext;
    }

    SOCKET getSocket()
    {
        return _socket;
    }

    OVERLAPPED* getOverlapped()
    {
        return _overlapped;
    }

    IOCP_OPC getOpCode()
    {
        std::cout << "IOCPClientContext::getOpCode() -> enter" << std::endl;
        IOCP_OPC opCode = OP_NONE;

        EnterCriticalSection(&_csOpCode);
        opCode = _opCode;
        LeaveCriticalSection(&_csOpCode);
        std::cout << "IOCPClientContext::getOpCode() -> leave" << std::endl;
        return opCode;
    }

    char *getNextWriteBufQueue(unsigned long &len)
    {
        std::cout << "IOCPClientContext::getNextWriteBufQueue() -> enter" << std::endl;
        char *buf = NULL;
        len = 0;

        EnterCriticalSection(&_csWriteBufQueue);

        if (_writeBufQueue.size() > 0)
        {
            buf = _writeBufQueue[0];
            len = _writeBufLengthQueue[0];

            std::vector<char*>::iterator iter;

            for (iter=_writeBufQueue.begin();iter!=_writeBufQueue.end();++iter)
            {
                _writeBufQueue.erase(iter);
                break;
            }

            std::vector<unsigned long>::iterator iter2;

            for (iter2=_writeBufLengthQueue.begin();iter2!=_writeBufLengthQueue.end();++iter2)
            {
                _writeBufLengthQueue.erase(iter2);
                break;
            }
        }

        LeaveCriticalSection(&_csWriteBufQueue);
        std::cout << "IOCPClientContext::getNextWriteBufQueue() -> leave" << std::endl;
        return buf;
    }

    int getTotalBytesTransfered()
    {
        return _totalBytesTransfered;
    }

    int getTotalBytesSent()
    {
        return _totalBytesSent;
    }

    int getTotalBytesRead()
    {
        return _totalBytesTransfered-_totalBytesSent;
    }

    void *getCustomContext()
    {
        return _customContext;
    }
private:
    CRITICAL_SECTION _csOpCode;
    CRITICAL_SECTION _csWriteBufQueue;
    SOCKET _socket;
    OVERLAPPED *_overlapped;
    HANDLE _hIOCP;
    IOCP_OPC _opCode;
    std::vector<char*> _writeBufQueue;
    std::vector<unsigned long> _writeBufLengthQueue;
    int _totalBytesTransfered;
    int _totalBytesSent;
    void *_customContext;
};

class IOCPServer
{
public:
    IOCPServer(unsigned short port, unsigned int maxInteractBytes);
    virtual ~IOCPServer();

    void start();
    void stop();

    void cleanUp();
    void deInit();
    static DWORD WINAPI AcceptThread(LPVOID lpvData);
    void acceptor();
    void AcceptConnection();
    static DWORD WINAPI WorkerThread(LPVOID lpvData);
    void worker();
    void addClient(IOCPClientContext *pClientContext);
    void removeClient(IOCPClientContext *pClientContext);
    void cleanClients();
    bool isOk();
protected:
    unsigned int _maxInteractBytes;

    virtual bool onSocketConnection(IOCPClientContext *pClientContext) = NULL;
    virtual void onSocketInput(IOCPClientContext *pClientContext, WSABUF *wsaBuf) = NULL;
//    virtual void onSocketOutput(IOCPClientContext *pClientContext, WSABUF *wsaBuf) = NULL;
    virtual void onSocketRemoved(IOCPClientContext *pClientContext, IOCP_CRR removeReason) = NULL;
private:
    SOCKET _listenerSocket;
    HANDLE _hShutdownEvent;
    unsigned int _threadCount;
    HANDLE *_hWorkerThreads;
    HANDLE _hAcceptThread;
    WSAEVENT _wsaAcceptEvent;
    CRITICAL_SECTION _csClientList;
    HANDLE _hIOCP;
    std::vector<IOCPClientContext*> _clients;
    bool _isOk;
    unsigned short _port;
};

#endif

#include <stdio.h>
#include <stdlib.h>
#include <conio.h>
#include <string.h>
#include <winsock2.h>
#include <vector>
#include <iostream>

#include "IOCPServer.h"

IOCPServer::IOCPServer(unsigned short port, unsigned int maxInteractBytes)
: _port(port), _maxInteractBytes(maxInteractBytes), _hShutdownEvent(NULL), _threadCount(0), _hWorkerThreads(NULL), _hAcceptThread(NULL), _hIOCP(NULL), _isOk(false)
{
    std::cout << "IOCPServer::IOCPServer() -> enter" << std::endl;
    SYSTEM_INFO si;

    GetSystemInfo(&si);
    _threadCount = WORKER_THREADS_PER_PROCESSOR * si.dwNumberOfProcessors;
    _hWorkerThreads = new HANDLE[_threadCount];
    InitializeCriticalSection(&_csClientList);
    _hShutdownEvent = CreateEvent(NULL, TRUE, FALSE, NULL);

    WSADATA wsaData;

    if (WSAStartup(MAKEWORD(2,2), &wsaData) != NO_ERROR)
        return;

    _hIOCP = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0);

    if (_hIOCP == NULL)
        return;

    struct sockaddr_in serverAddr;

    _listenerSocket = WSASocket(AF_INET, SOCK_STREAM, 0, NULL, 0, WSA_FLAG_OVERLAPPED);

    if (_listenerSocket == INVALID_SOCKET) 
        goto error;

    ZeroMemory((char*)&serverAddr, sizeof(serverAddr));
    serverAddr.sin_family = AF_INET;
    serverAddr.sin_addr.s_addr = INADDR_ANY;
    serverAddr.sin_port = htons(_port);

    if (bind(_listenerSocket, (struct sockaddr*)&serverAddr, sizeof(serverAddr)) == SOCKET_ERROR) 
        goto error;

    if (listen(_listenerSocket, SOMAXCONN) == SOCKET_ERROR)
        goto error;

    if ((_wsaAcceptEvent = WSACreateEvent()) == WSA_INVALID_EVENT)
        goto error;

    if (WSAEventSelect(_listenerSocket, _wsaAcceptEvent, FD_ACCEPT) == SOCKET_ERROR)
    {
        WSACloseEvent(_wsaAcceptEvent);
        goto error;
    }

    _isOk = true;
    std::cout << "IOCPServer::IOCPServer() -> leave" << std::endl;
    return;
error:
    closesocket(_listenerSocket);
    deInit();
    std::cout << "IOCPServer::IOCPServer() -> leave error" << std::endl;
}

IOCPServer::~IOCPServer()
{}

void IOCPServer::start()
{
    std::cout << "IOCPServer::start() -> enter" << std::endl;
    DWORD dwThreadId;

    for (unsigned int i=0;i<_threadCount;i++)
        _hWorkerThreads[i] = CreateThread(0, 0, WorkerThread, this, 0, &dwThreadId);

    _hAcceptThread = CreateThread(0, 0, AcceptThread, this, 0, &dwThreadId);
    std::cout << "IOCPServer::start() -> leave" << std::endl;
}

void IOCPServer::stop()
{
    std::cout << "IOCPServer::stop() -> enter" << std::endl;
    cleanUp();
    closesocket(_listenerSocket);
    deInit();
    std::cout << "IOCPServer::stop() -> leave" << std::endl;
}

bool IOCPServer::isOk()
{
    return _isOk;
}

void IOCPServer::cleanUp()
{
    std::cout << "IOCPServer::cleanUp() -> enter" << std::endl;
    SetEvent(_hShutdownEvent);
    WaitForSingleObject(_hAcceptThread, INFINITE);

    for (unsigned int i=0;i<_threadCount;i++)
        PostQueuedCompletionStatus(_hIOCP, 0, (DWORD)NULL, NULL);

    WaitForMultipleObjects(_threadCount, _hWorkerThreads, TRUE, INFINITE);
    WSACloseEvent(_wsaAcceptEvent);
    cleanClients();
    std::cout << "IOCPServer::cleanUp() -> leave" << std::endl;
}

void IOCPServer::deInit()
{
    //Delete the Client List Critical Section.
    DeleteCriticalSection(&_csClientList);

    //cleanUp IOCP.
    CloseHandle(_hIOCP);

    //Clean up the event.
    CloseHandle(_hShutdownEvent);

    //Clean up memory allocated for the storage of thread handles
    delete[] _hWorkerThreads;

    //cleanUp Winsock
    WSACleanup();
}

//This thread will look for accept event
DWORD WINAPI IOCPServer::AcceptThread(LPVOID lpvData)
{
    ((IOCPServer*)lpvData)->acceptor();
    return 0;
}

void IOCPServer::acceptor()
{
    std::cout << "IOCPServer::acceptor() -> enter" << std::endl;
    WSANETWORKEVENTS WSAEvents;

    //Accept thread will be around to look for accept event, until a Shutdown event is not Signaled.
    while(WaitForSingleObject(_hShutdownEvent, 0) != WAIT_OBJECT_0)
    {
        if (WSAWaitForMultipleEvents(1, &_wsaAcceptEvent, FALSE, WAIT_TIMEOUT_INTERVAL, FALSE) != WSA_WAIT_TIMEOUT)
        {
            WSAEnumNetworkEvents(_listenerSocket, _wsaAcceptEvent, &WSAEvents);

            if ((WSAEvents.lNetworkEvents & FD_ACCEPT) && (0 == WSAEvents.iErrorCode[FD_ACCEPT_BIT]))
                AcceptConnection();
        }
    }
    std::cout << "IOCPServer::acceptor() -> leave" << std::endl;
}

//This function will process the accept event
void IOCPServer::AcceptConnection()
{
    std::cout << "IOCPServer::AcceptConnection() -> enter" << std::endl;
    sockaddr_in clientAddr;
    int clientAddrLength = sizeof(clientAddr);

    //Accept remote connection attempt from the client
    SOCKET acceptedSocket = accept(_listenerSocket, (sockaddr*)&clientAddr, &clientAddrLength);

    if (acceptedSocket == INVALID_SOCKET) // error while accepting socket?
    {
        std::cout << "IOCPServer::AcceptConnection() -> leave error 1" << std::endl;
        return; // WSAGetLastError()
    }

    // Client IP: inet_ntoa(clientAddr.sin_addr);

    IOCPClientContext *pClientContext  = new IOCPClientContext(acceptedSocket, _hIOCP);

    addClient(pClientContext);

    HANDLE hClientIOCP = CreateIoCompletionPort((HANDLE)pClientContext->getSocket(), _hIOCP, (DWORD)pClientContext, 1);

    if (hClientIOCP == NULL) // error creating IOCP?
    {
        removeClient(pClientContext);
        onSocketRemoved(pClientContext, IOCP_CRR_CREATE_ERROR);
        delete pClientContext;
        std::cout << "IOCPServer::AcceptConnection() -> leave error 2" << std::endl;
        return;
    }

    // post first WSARecv or WSASend in onSocketConnection, so workers can start to work with this client
    if (!onSocketConnection(pClientContext))
    {
        removeClient(pClientContext);
        delete pClientContext;
        std::cout << "IOCPServer::AcceptConnection() -> leave error 3" << std::endl;
        return;
    }

    std::cout << "IOCPServer::AcceptConnection() -> leave" << std::endl;
}

//Worker thread will service IOCP requests
DWORD WINAPI IOCPServer::WorkerThread(LPVOID lpvData)
{    
    ((IOCPServer*)lpvData)->worker();
    return 0;
}

void IOCPServer::worker()
{
    std::cout << "IOCPServer::worker() -> enter" << std::endl;
    LPVOID lpvContext = NULL;
    OVERLAPPED *pOverlapped = NULL;
    IOCPClientContext *pClientContext = NULL;
    DWORD dwBytesTransfered = 0;
    int nBytesRecv = 0;
    int nBytesSent = 0;
    DWORD dwBytes = 0;
    DWORD dwFlags = 0;

    //Worker thread will be around to process requests, until a Shutdown event is not Signaled.
    while (WaitForSingleObject(_hShutdownEvent, 0) != WAIT_OBJECT_0)
    {
        BOOL bReturn = GetQueuedCompletionStatus(_hIOCP, &dwBytesTransfered, (LPDWORD)&lpvContext, &pOverlapped, INFINITE);

        if (lpvContext == NULL) // shutdown ?
            break;

        //Get the client context
        pClientContext = (IOCPClientContext*)lpvContext;

        if ((FALSE == bReturn) || ((TRUE == bReturn) && (0 == dwBytesTransfered)))
        {
            //Client connection gone, remove it.
            removeClient(pClientContext);
            onSocketRemoved(pClientContext, IOCP_CRR_GONE);
            delete pClientContext;
            continue;
        }

        WSABUF *wsaBuf = new WSABUF;
        SOCKET socket = pClientContext->getSocket();
        OVERLAPPED *overlapped = pClientContext->getOverlapped();
        IOCP_OPC opCode = pClientContext->getOpCode();

        // when a client connect you have to set OP_READ or OP_WRITE, so worker process something for the client
        switch (opCode)
        {
        case OP_READ:
            pClientContext->setOpCode(OP_WRITE);
            wsaBuf->buf = new char[_maxInteractBytes];
            nBytesRecv = WSARecv(socket, wsaBuf, 1, &dwBytes, &dwFlags, overlapped, NULL);

            if (nBytesRecv == SOCKET_ERROR)
            {
                if (WSAGetLastError() != WSA_IO_PENDING)
                {
                    removeClient(pClientContext);
                    onSocketRemoved(pClientContext, IOCP_CRR_RECV_ERROR);
                    delete pClientContext;
                }

                continue;
            }

            onSocketInput(pClientContext, wsaBuf);
            break;
        case OP_WRITE:
            pClientContext->setOpCode(OP_READ);

            for (;;)
            {
                wsaBuf->buf = pClientContext->getNextWriteBufQueue(wsaBuf->len);

                if (wsaBuf->buf == NULL)
                    break;

                if (wsaBuf->len > _maxInteractBytes)
                    wsaBuf->len = _maxInteractBytes;

                nBytesSent = WSASend(socket, wsaBuf, 1, &dwBytes, dwFlags, overlapped, NULL);

                if (nBytesSent == SOCKET_ERROR)
                {
                    if (WSAGetLastError() != WSA_IO_PENDING)
                    {
                        removeClient(pClientContext);
                        onSocketRemoved(pClientContext, IOCP_CRR_SEND_ERROR);
                        delete pClientContext;
                    }

                    break;
                }
            }

            break;
        default:
            break;
        }        
    }

    std::cout << "IOCPServer::worker() -> leave" << std::endl;
}

//Store client related information in a vector
void IOCPServer::addClient(IOCPClientContext *pClientContext)
{
    std::cout << "IOCPServer::addClient() -> enter" << std::endl;
    EnterCriticalSection(&_csClientList);
    _clients.push_back(pClientContext);
    LeaveCriticalSection(&_csClientList);
    std::cout << "IOCPServer::addClient() -> leave" << std::endl;
}

//This function will allow to remove one single client out of the list
void IOCPServer::removeClient(IOCPClientContext *pClientContext)
{
    std::cout << "IOCPServer::removeClient() -> enter" << std::endl;
    EnterCriticalSection(&_csClientList);

    std::vector<IOCPClientContext*>::iterator iter;

    //Remove the supplied ClientContext from the list and release the memory
    for (iter=_clients.begin();iter!=_clients.end();++iter)
    {
        if (pClientContext == *iter)
        {
            _clients.erase(iter);
            break;
        }
    }

    LeaveCriticalSection(&_csClientList);
    std::cout << "IOCPServer::removeClient() -> leave" << std::endl;
}

void IOCPServer::cleanClients()
{
    std::cout << "IOCPServer::cleanClients() -> enter" << std::endl;
    EnterCriticalSection(&_csClientList);

    std::vector<IOCPClientContext*>::iterator iter;

    for (iter=_clients.begin();iter!=_clients.end();++iter)
        delete *iter;

    _clients.clear();

    LeaveCriticalSection(&_csClientList);
    std::cout << "IOCPServer::cleanClients() -> leave" << std::endl;
}

class MyIOCPServer : public IOCPServer
{
public:
    MyIOCPServer(unsigned int port, unsigned int maxRecvBytes)
    : IOCPServer(port, maxRecvBytes)
    {
        if (isOk())
            start();
    }

    virtual ~MyIOCPServer()
    {
        if (isOk())
            stop();
    }

    // damn use an initial WSARecv/WSASend so worker can start to work with this client
    bool onSocketConnection(IOCPClientContext *pClientContext)
    {
        // you can add a custom client context here to the pClientContext(IOCPClientContext::setCustomClientContext()), IOCPClientContext does not delete _customContext,  it is stored as void*, when onSocketRemoved is caled you should cast it to the type it is(you should know it) so d'tor is called

        SOCKET socket = pClientContext->getSocket();
        OVERLAPPED *overlapped = pClientContext->getOverlapped();

        WSABUF *wsaBuf = new WSABUF;
        DWORD dwBytes = 0;
        DWORD dwFlags = 0;

        pClientContext->setOpCode(OP_WRITE);

        /*wsaBuf->buf = "fsddsf"; // damn do something
        wsaBuf->len = strlen("fsddsf");

        int bytesSent = WSASend(socket, wsaBuf, 1, &dwBytes, dwFlags, overlapped, NULL);

        if (bytesSent == SOCKET_ERROR)
        {
            std::cout << "socket error on initial send!" << std::endl;
            return false; // it returns false, so the pClientContext is removed and deleted
        }*/

        wsaBuf->buf = new char[_maxInteractBytes];
        wsaBuf->len = _maxInteractBytes;

        int bytesRead = WSARecv(socket, wsaBuf, 1, &dwBytes, &dwFlags, overlapped, NULL);

        if (bytesRead == SOCKET_ERROR)
            return false; // it returns false, so the pClientContext is removed and deleted

        std::cout << "Echo: " << wsaBuf->buf << std::endl;
        pClientContext->addToWriteBufQueue(wsaBuf->buf, wsaBuf->len);
        return true;
    }

    void onSocketInput(IOCPClientContext *pClientContext, WSABUF *wsaBuf)
    {
        std::cout << "on socket input" << std::endl;
        std::cout << wsaBuf->buf << std::endl;
        pClientContext->addToWriteBufQueue(wsaBuf->buf, wsaBuf->len);
    }

    void onSocketRemoved(IOCPClientContext *pClientContext, IOCP_CRR removeReason)
    {
        std::cout << "on socket removed" << std::endl;

        if (pClientContext == NULL)
            return;

        switch (removeReason)
        {
        case IOCP_CRR_CREATE_ERROR:
            std::cout << "IOCP_CRR::IOCP_CRR_CREATE_ERROR" << std::endl;
            break;
        case IOCP_CRR_GONE:
            std::cout << "IOCP_CRR::IOCP_CRR_GONE" << std::endl;
            break;
        case IOCP_CRR_RECV_ERROR:
            std::cout << "IOCP_CRR::IOCP_CRR_RECV_ERROR" << std::endl;
            break;
        case IOCP_CRR_SEND_ERROR:
            std::cout << "IOCP_CRR::IOCP_CRR_RECV_ERROR" << std::endl;
            break;
        }
    }
};

int main()
{
    MyIOCPServer *iocpServer = new MyIOCPServer(2106, 65535);

    while (!kbhit())
        Sleep(1);

    delete iocpServer;
    system("pause");

    return 0;
}

Ich denke nicht das ihr den Sourcecode des echo clienten braucht,
wie ihr sehen könnt hat die ServerKlasse 3 virtual Methoden die überschrieben werden
müssen um connections,input,lost zu handeln. In diesem macht der Server bei einer
neuen Verbindung ein initial WSARecv um Daten vom client zu bekommen. Nun fangen
die worker Threads an mit dem Klienten uzu arbeiten(ja wie sollte es anders sein
mit GetQueuedCompletionStatus() ^^), was nur geschiet wenn ein initial
WSARecv/WSASend geschiet. Bei jedem WSARecv/WSASend resultiert
GetQueuedCompletionStatus(), ganz gleich ob diese fehlschlagen oder erfolgreich sind.

Oder besser gesagt die aktion wird zum queue hinzugefügt und der queue wird der
Reihenfolge abgearbeitet. Soweit alles richtig? Sagt mir bitte wenn ich was missverstanden habe. Und wenn möglich bitte stellung zu meinen 2 problemen nehem

P.S: Cool du hast dir alles durchgelesen ^^

Best wishes
FBIagent