I have QSplitter set as the central widget:

MainWindow::MainWindow(QWidget *parent)
    : QMainWindow(parent),
      mdiArea(new QMdiArea)
{
    QWidget *widget = new QWidget;
    widget->setMinimumSize(100, 100);

    QSplitter *splitter = new QSplitter;
    splitter->addWidget(mdiArea);
    splitter->addWidget(widget);

    setCentralWidget(splitter);

    createActions();
}

void MainWindow::createSubwin()
{
    QWidget *subwin = new QWidget(mdiArea);
    subwin->setWindowTitle("Subwindow");
    subwin->setMinimumSize(100, 100);
    mdiArea->addSubWindow(subwin);

    subwin->show();
}

void MainWindow::createActions()
{
    QAction *actSub = new QAction("Add subwindow", this);
    connect(actSub, SIGNAL(triggered()), SLOT(createSubwin()));

    QMenu *winMenu = menuBar()->addMenu("Windows");
    winMenu->addAction(actSub);
}

When I press maximize button of subwindow, the subwindow covers entire main window. Is there any way to prevent such behaviour and make subwindow occupy all the space of QMdiArea instead?

UPD: It looks like that the problem occurs only when at least one menu in main window's QMenuBar is present. Without menuBar everything works as expected:

So, I cannot reproduce OPs claim—it works on my side.

My platform: Windows 10, VS2019, Qt5.15

I enhanced the first MCVE a bit to see how it works if MDI sub-windows are created after qWinMain.show() (what's expected as the usual case).

#include <QtWidgets>

// main application

int main(int argc, char **argv)
{
  qDebug() << "Qt Version:" << QT_VERSION_STR;
  QApplication app(argc, argv);
  // setup GUI
  QMainWindow qWinMain;
  qWinMain.setWindowTitle("QMainWindow - MDI - Dock");
  qWinMain.resize(640, 480);
  // MDI
  QMdiArea qMDI;
  qWinMain.setCentralWidget(&qMDI);
  // Dock
  QDockWidget qWinDock;
  qWinDock.setWindowTitle("Dock");
  QWidget qDock;
  QVBoxLayout qVBoxDock;
  QPushButton qBtnNewMDISubWin("New Sub-Window");
  qVBoxDock.addWidget(&qBtnNewMDISubWin);
  qDock.setLayout(&qVBoxDock);
  qWinDock.setWidget(&qDock);
  qWinMain.addDockWidget(Qt::RightDockWidgetArea, &qWinDock);
  // create sub-window
  int i = 0;
  auto createSubWin = [&]() {
      i;
    QLabel* pQWinSub = new QLabel(QString("MDI Sub-Window\nwidget %1").arg(i));
    pQWinSub->setWindowTitle(QString("MDI Sub-Window %1").arg(i));
    qMDI.addSubWindow(pQWinSub);
    pQWinSub->show();
  };
  // install signal handlers
  QObject::connect(&qBtnNewMDISubWin, &QPushButton::clicked,
    createSubWin);
  // runtime loop
  qWinMain.show();
  return app.exec();
}

Output:

It still works on my side as expected.

Note:

I had to add the explicit pQWinSub->show(); after qMDI.addSubWindow(pQWinSub); (which was not necessary in the first MCVE). However, this is exactly how it's done by OP's code.

OPs reply:

It turns out that the problem occurs only when menuBar is present

Oha. How comes?

I extended my MCVE again to add a menu bar:

#include <QtWidgets>

// main application

int main(int argc, char **argv)
{
  qDebug() << "Qt Version:" << QT_VERSION_STR;
  QApplication app(argc, argv);
  // setup GUI
  QMainWindow qWinMain;
  qWinMain.setWindowTitle("QMainWindow - MDI - Dock");
  qWinMain.resize(640, 480);
  // menu
  QMenuBar qMenuMain;
  QAction qCmdFile("File");
  QMenu qMenuFile;
  QAction qCmdFileNew("New");
  qMenuFile.addAction(&qCmdFileNew);
  qCmdFile.setMenu(&qMenuFile);
  qMenuMain.addAction(&qCmdFile);
  qWinMain.setMenuBar(&qMenuMain);
  // MDI
  QMdiArea qMDI;
  qWinMain.setCentralWidget(&qMDI);
  // Dock
  QDockWidget qWinDock;
  qWinDock.setWindowTitle("Dock");
  QWidget qDock;
  QVBoxLayout qVBoxDock;
  QPushButton qBtnNewMDISubWin("New Sub-Window");
  qVBoxDock.addWidget(&qBtnNewMDISubWin);
  qDock.setLayout(&qVBoxDock);
  qWinDock.setWidget(&qDock);
  qWinMain.addDockWidget(Qt::RightDockWidgetArea, &qWinDock);
  // create sub-window
  int i = 0;
  auto createSubWin = [&]() {
      i;
    QLabel* pQWinSub = new QLabel(QString("MDI Sub-Window\nwidget %1").arg(i));
    pQWinSub->setWindowTitle(QString("MDI Sub-Window %1").arg(i));
    pQWinSub->setFrameShape(QFrame::Box);
    qMDI.addSubWindow(pQWinSub);
    pQWinSub->show();
  };
  // install signal handlers
  QObject::connect(&qCmdFileNew, &QAction::triggered,
    createSubWin);
  QObject::connect(&qBtnNewMDISubWin, &QPushButton::clicked,
    createSubWin);
  // runtime loop
  qWinMain.show();
  return app.exec();
}

Output:

Note:

I partly agree with OP:

Yes, the look of the maximized MDI is a bit different now. It looks like it occupies the whole client area of the main window but…

…the dock widget is still visible. I added a box to the QLabel (the top widget in the MDI sub-window) to illustrate this. In fact, the sub-window still occupies the central widget only (regardless what the look of its title bar suggests).