My code

… was written in C++11 and can be compiled with G++, Clang++, Visual C++. You can download it, too. Or just jump to my GitHub repository.

       
#include <iostream>
       
#include <vector>
       
#include <map>
       
 
       
const auto MaxSize = 10; // no grids larger than 10x10 (6x6 is already super-slow in brute-force mode)
       
auto size   = 5;        // that's m from the problem statement
       
auto cutout = 3;        // that's n from the problem statement
       
 
       
const auto Modulo  = 76543217;
       
 
       
// indicates a cell that hasn't be filled yet

       
const unsigned int Unknown = 0;
       
unsigned int cells[MaxSize][MaxSize] = { Unknown };
       
 
       
// add "number" and all smaller numbers to "cells", return how many combinations where found

       
unsigned long long bruteForce(unsigned int number)
       
{
       
  // successfully filled all cells ?
       
  if (number == 0)
       
  {
       
    static unsigned long long progress = 0;
       
    if (++progress % (1 << 24) == 0)
       
      std::cout << (progress * 100.0 / 406029023400) << "%" << std::endl;
       
    return 1;
       
  }
       
 
       
  unsigned long long result = 0;
       
 
       
  // iterate over all free spots where the left and lower neighbor are bigger (or there is no neighbor => border)
       
  for (auto y = 0; y < size; y++)
       
  {
       
    // avoid the cut-out area
       
    auto maxX = size;
       
    if (y < cutout)
       
      maxX = size - cutout;
       
 
       
    for (auto x = 0; x < maxX; x++)
       
    {
       
      // cell previously filled ?
       
      if (cells[x][y] != Unknown)
       
        continue;
       
 
       
      // all bigger numbers were already processed, so there's no gap allowed on the left or bottom
       
 
       
      // left   neighbor must exist, or there's the grid's border
       
      if (x > 0 && cells[x-1][y] == Unknown)
       
        continue;
       
      // bottom neighbor must exist, or there's the grid's border
       
      if (y < size - 1 && cells[x][y+1] == Unknown)
       
        continue;
       
 
       
      // okay, found an empty cell !
       
      cells[x][y] = number;
       
      // go deeper
       
      result += bruteForce(number - 1);
       
 
       
      // undo current step and keep looking for other configurations
       
      cells[x][y] = Unknown;
       
    }
       
  }
       
 
       
  return result % Modulo;
       
}
       
 
       
// my "skyscraper" algorithm

       
typedef std::vector<unsigned short> HeightMap;
       
unsigned long long slow(unsigned int number, HeightMap& height)
       
{
       
  // successfully filled all cells ?
       
  if (number == 1)
       
    return 1;
       
 
       
  // memoize
       
  static std::map<HeightMap, unsigned long long> cache;
       
  auto lookup = cache.find(height);
       
  if (lookup != cache.end())
       
    return lookup->second;
       
 
       
  // find a column that isn't full and is lower than its left neighbor
       
  unsigned long long result = 0;
       
  for (auto i = 0; i < size; i++)
       
  {
       
    // maximum height
       
    auto maxHeight = size;
       
    if (i >= size - cutout)
       
      maxHeight -= cutout;
       
 
       
    if (height[i] < maxHeight)
       
      // lower than its left neighbor ?
       
      if (i == 0 || height[i] < height[i - 1])
       
      {
       
        height[i]++;
       
        result += slow(number - 1, height);
       
        height[i]--;
       
      }
       
  }
       
 
       
  result %= Modulo;
       
  cache[height] = result;
       
  return result;
       
}
       
 
       
int main()
       
{
       
  std::cin >> size >> cutout;
       
  // for live tests only
       
  if (size > MaxSize || size < cutout)
       
    return 1;
       
 
       
  // the highest number is always located in the lower-right corner
       
  auto highestNumber = size * size - cutout * cutout;
       
 
       
  // brute force
       
  //std::cout << bruteForce(highestNumber) << std::endl;
       
 
       
  // significantly faster, but still too slow to solve the problem (it easily solves all examples)
       
  HeightMap height(size, 0);
       
  std::cout << slow(highestNumber, height) << std::endl;
       
 
       
  return 0;
       
}

This solution contains 22 empty lines, 23 comments and 3 preprocessor commands.