/* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim:set sw=2 sts=2 et cin: */
/*
 * This file is part of the MUSE Instrument Pipeline
 * Copyright (C) 2005-2017 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

/*----------------------------------------------------------------------------*
 *                             Includes                                       *
 *----------------------------------------------------------------------------*/
#if HAVE_POPEN && HAVE_PCLOSE
#define _DEFAULT_SOURCE
#define _BSD_SOURCE /* force popen/pclose, mkdtemp definitions from stdio/stdlib */
#endif
#include <cpl.h>
#include <string.h>
#undef __USE_MISC /* don't want y1 */
#include <math.h>

#include "muse_tracing.h"
#include "muse_instrument.h"

#include "muse_cplwrappers.h"
#include "muse_pfits.h"
#include "muse_quadrants.h"
#include "muse_utils.h"

/*----------------------------------------------------------------------------*/
/**
 * @defgroup muse_tracing      Tracing
 */
/*----------------------------------------------------------------------------*/

/**@{*/

/* strings corresponding to the muse_trace_poly entries in muse_tracing.h */
static const char *muse_trace_poly_strings[] = {
  "central",
  "left",
  "right"
};

static void muse_trace_plot_located_slices(cpl_vector *, cpl_vector *, double, double, double, const unsigned char);

/*---------------------------------------------------------------------------*/
/**
  @brief   Create a vector containing a representative horizontal image cut.
  @param   aImage   the input image, most likely a master flat-field
  @param   aNRows   number of rows to collapse in each quadrant
  @return  the vertically collapsed data as cpl_vector or NULL on error

  Use two data samples from a bit below and above the vertical image center,
  and create a vector containing the maximum of both samples. This minimizes the
  possible influence of dark columns onto the output vector.

  @error{set CPL_ERROR_NULL_INPUT\, return NULL,
         aImage or its data component are NULL}
 */
/*---------------------------------------------------------------------------*/
static cpl_vector *
muse_trace_horizontal_cut(const muse_image *aImage, unsigned int aNRows)
{
  cpl_ensure(aImage && aImage->data, CPL_ERROR_NULL_INPUT, NULL);

  /* set the vertical midpoint by looking at the top quadrant boundary, of the *
   * lower left quadrant but preset to be the vertical center of the image     */
  cpl_size ymid = cpl_image_get_size_y(aImage->data) / 2,
           *w = muse_quadrants_get_window(aImage, 1);
  if (w) { /* can fail if a test master flat without headers was passed in */
    ymid = w[3]; /* top of this bottom quadrant */
    cpl_free(w);
  }
  int y1 = ymid - 1 - aNRows,
      y2 = ymid - 1,
      y3 = ymid + 1,
      y4 = ymid + 1 + aNRows;
  cpl_msg_debug(__func__, "IFU %hhu: ymid=%"CPL_SIZE_FORMAT", region=%d..%d, %d..%d",
                muse_utils_get_ifu(aImage->header), ymid, y1, y2, y3, y4);

  /* collapse both regions, vertically and separately */
  int nx = cpl_image_get_size_x(aImage->data);
  cpl_image *tmp1 = cpl_image_collapse_window_create(aImage->data, 1, y1, nx, y2, 0),
            *tmp2 = cpl_image_collapse_window_create(aImage->data, 1, y3, nx, y4, 0);
  /* normalize, to make them comparable despite possible gain differences */
  cpl_image_normalise(tmp1, CPL_NORM_MEAN);
  cpl_image_normalise(tmp2, CPL_NORM_MEAN);
  /* construct max image to exclude bad columns */
  cpl_image *tmax = cpl_image_new(nx, 1, CPL_TYPE_FLOAT);
  int i;
  for (i = 1; i <= nx; i++) {
    int err;
    cpl_image_set(tmax, i, 1,
                  fmax(cpl_image_get(tmp1, i, 1, &err),
                       cpl_image_get(tmp2, i, 1, &err)));
  } /* for i (horizontal pixels) */
#if 0 /* DEBUG */
  cpl_image_save(tmp1, "trace_tmp1.fits", CPL_TYPE_FLOAT, NULL, CPL_IO_CREATE);
  cpl_image_save(tmp2, "trace_tmp2.fits", CPL_TYPE_FLOAT, NULL, CPL_IO_CREATE);
  cpl_image_save(tmax, "trace_tmax.fits", CPL_TYPE_FLOAT, NULL, CPL_IO_CREATE);
  cpl_msg_debug(__func__, "Saved collapsed rows to trace_{tmp1,tmp2,tmax}.fits");
#endif
  cpl_image_delete(tmp1);
  cpl_image_delete(tmp2);

  cpl_vector *cut = cpl_vector_new_from_image_row(tmax, 1);
  cpl_image_delete(tmax);
  return cut;
} /* muse_trace_horizontal_cut() */

/*---------------------------------------------------------------------------*/
/**
  @brief   Find all slice midpoints across a CCD.
  @param   aRowVec     cpl_vector containing the row data from the (middle)
                       of the CCD
  @param   aNSlices    number of slices in the MUSE CCD
  @param   aFrac       start fraction of median used to identify edges
  @param   aIFU        the IFU number
  @return  the slit centers as cpl_vector * or NULL in case of failure
  @remark  The first slice is assumed to be fully located within the first
           aFirstSlice pixels, a slice is assumed to be no wider than
           aMaxWidth pixels.

  Go along the input data vector to search for the two outer edges of the first
  slice. An edge is detected by comparing the data value to the median value
  along the data vector. If the difference is larger than aFrac times the
  median of the data vector, an edge was found. This procedure is repeated for
  all aNSlices. If not all slices are located at least a certain distance apart,
  the routine is rerun with a decreased aFrac.
  The returned midpoints are the arithmetic mean of the two edge positions for
  each slice.

  @error{set CPL_ERROR_NULL_INPUT\, return NULL, aRowVec is NULL}
  @error{set CPL_ERROR_ILLEGAL_INPUT\, return NULL, aFrac is invalid (<0 or >1)}
  @error{set MUSE_ERROR_SLICE_LEFT_MISSING\, return NULL,
         search for first slice (left-edge) failed}
  @error{set MUSE_ERROR_SLICE_RIGHT_MISSING\, return NULL,
         search for first slice (right-edge) failed}
  @error{set MUSE_ERROR_SLICE_EDGE_MISSING\, return NULL,
         search for an edge of any of the other slices failed (less than 96 edges are found)}
  @error{set CPL_ERROR_ACCESS_OUT_OF_RANGE\, return NULL,
         initial slice was found to be too narrow or too wide}
 */
/*---------------------------------------------------------------------------*/
cpl_vector *
muse_trace_locate_slices(cpl_vector *aRowVec, const unsigned short aNSlices,
                         double aFrac, const unsigned char aIFU)
{
  cpl_ensure(aRowVec, CPL_ERROR_NULL_INPUT, NULL);
  cpl_ensure(aFrac > 0. && aFrac < 1., CPL_ERROR_ILLEGAL_INPUT, NULL);
  cpl_vector *centers = cpl_vector_new(aNSlices),
             *widths = cpl_vector_new(aNSlices);
  double f = aFrac;

  /* Loop through the actual slice location routine until all slices      *
   * monotonically increase in x-direction with a large enough step size. */
  while (1) {
    double median = cpl_vector_get_median_const(aRowVec),
           mdev = muse_cplvector_get_adev_const(aRowVec, median),
           detlimit = f*median; /* the detection limit */

    cpl_msg_debug(__func__, "median=%f, mdev=%f, fraction=%f --> edge detection"
                  " limit=%f (IFU %hhu)", median, mdev, f, detlimit, aIFU);

    /* find first edge from the left: assume it lies within the first *
     * kMuseSliceSearchRegion pix                                     */
    double ledge = 0., redge = 0.;
    int i;
    for (i = 0; i <= kMuseSliceSearchRegion; i++) {
      if (cpl_vector_get(aRowVec, i) >= detlimit) {
        ledge = i - 0.5; /* the real edge is probably in between... */
        break;
      }
    }
    if (i == kMuseSliceSearchRegion) {
      cpl_msg_error(__func__, "Search for first slice (left-edge) failed in IFU"
                    " %hhu", aIFU);
      cpl_vector_delete(centers);
      centers = NULL;
      break;
    }

    /* find the corresponding right edge, should be within the next *
     * kMuseSliceMaxWidth pix                                       */
    for (i = ledge + 1.5; i <= ledge + kMuseSliceMaxWidth; i++) {
      if (cpl_vector_get(aRowVec, i) <= detlimit) {
        redge = i - 0.5;
        break;
      }
    }
    if (i == ledge + kMuseSliceMaxWidth) {
      cpl_msg_error(__func__, "Search for first slice (right-edge) failed in "
                    "IFU %hhu", aIFU);
      cpl_vector_delete(centers);
      centers = NULL;
      break;
    }

    double width = redge - ledge;
    if (width < kMuseSliceLoLikelyWidth) {
      cpl_msg_error(__func__, "Initial slice is too narrow (%.2f pix, %.1f..%.1f)"
                    " -> search failed in IFU %hhu", width, ledge, redge, aIFU);
      cpl_error_set(__func__, CPL_ERROR_ACCESS_OUT_OF_RANGE);
      cpl_vector_delete(centers);
      centers = NULL;
      break;
    }
    if (width > kMuseSliceHiLikelyWidth) {
      cpl_msg_error(__func__, "Initial slice is too wide (%.2f pix, %.1f..%.1f)"
                    " -> search failed in IFU %hhu", width, ledge, redge, aIFU);
      cpl_error_set(__func__, CPL_ERROR_ACCESS_OUT_OF_RANGE);
      cpl_vector_delete(centers);
      centers = NULL;
      break;
    }

    /* derive and store the first midpoint (the above are indices!) */
    cpl_vector_set(centers, 0, round((ledge + redge) / 2.) + 1);
    cpl_vector_set(widths, 0, width);

    /* now find centers for the remaining slices in the same way */
    unsigned short j;
    for (j = 1; j < aNSlices; j++) {
      for (i = redge + 1.5; i <= redge + kMuseSliceMaxWidth; i++) {
        if (cpl_vector_get(aRowVec, i) >= detlimit) {
          ledge = i - 0.5;
          break;
        }
      }
      if (i == redge + kMuseSliceMaxWidth) {
        cpl_msg_error(__func__, "Search for slice %hu (left-edge) failed in IFU"
                      " %hhu", j, aIFU);
        cpl_vector_delete(centers);
        cpl_vector_delete(widths);
        return NULL;
      }

      for (i = ledge + 1.5; i <= ledge + kMuseSliceMaxWidth; i++) {
        if (cpl_vector_get(aRowVec, i) <= detlimit) {
          redge = i - 0.5;
          break;
        }
      }
      if (i == ledge + kMuseSliceMaxWidth) {
        cpl_msg_error(__func__, "Search for slice %hu (right-edge) failed in"
                      " IFU %hhu", j, aIFU);
        cpl_vector_delete(centers);
        cpl_vector_delete(widths);
        return NULL;
      }

      /* check that these values make sense, but don't fail */
      width = redge - ledge;
      cpl_vector_set(widths, j, width);
#if 0
      cpl_msg_debug(__func__, "slice %hu: left=%f, right=%f --> width %f, center %f",
                    j+1, ledge, redge, width, (ledge + redge) / 2.);
#endif
      /* set output, converting from vector indices to pixel positions */
      cpl_vector_set(centers, j, round((ledge + redge) / 2.) + 1);
    } /* for j (all slices after the first one) */

    /* possibly plot the result */
    char *doplot = getenv("MUSE_PLOT_TRACE");
    if (doplot && (atoi(doplot) & 0x1)) {
      muse_trace_plot_located_slices(aRowVec, centers, median, mdev, detlimit,
                                     aIFU);
    }

    int failures = 0;
    for (i = 1; i < cpl_vector_get_size(centers); i++) {
      double step = cpl_vector_get(centers, i) - cpl_vector_get(centers, i-1);
      if (step < kMuseSliceLoLikelyWidth) {
        failures++;
      }
    }

    /* if there were no misdetections, we can break the loop */
    if (!failures) {
      break;
    }

    /* decrease the fraction by a bit and try again */
    f /= 1.2;

    if (f < DBL_EPSILON) {
      cpl_msg_error(__func__, "Still detected %d unlikely slice locations, but "
                    "the cut-off fraction has become unrealistically small in "
                    "IFU %hhu (initial %f, now %f)", failures, aIFU, aFrac, f);
      break;
    }
  } /* while 1 */

  /* if nothing fatal occured, check widths of all slices and output warnings */
  int i, n = !centers ? -1 : cpl_vector_get_size(widths);
  for (i = 0; i < n; i++) {
    float width = cpl_vector_get(widths, i);
    if (width < kMuseSliceLoLikelyWidth) {
      cpl_msg_warning(__func__, "From the initial guess, slice %d appears to be"
                      " only %f pix wide in IFU %hhu, please cross-check!", i+1,
                      width, aIFU);
    }
    if (width > kMuseSliceHiLikelyWidth) {
      cpl_msg_warning(__func__, "From the initial guess, slice %d appears to be"
                      " very wide in IFU %hhu (%f pix), please cross-check!",
                      i+1, aIFU, width);
    }
    if (i >= 1) {
      double step = cpl_vector_get(centers, i) - cpl_vector_get(centers, i-1);
      if (step < kMuseSliceLoLikelyWidth) {
        cpl_msg_warning(__func__, "Slice %d is only %.2f pix farther than the "
                        "previous one in IFU %hhu!", i + 1, step, aIFU);
      }
    }
  } /* for i (slices) */
  cpl_vector_delete(widths);

  return centers;
} /* muse_trace_locate_slices() */

/*---------------------------------------------------------------------------*/
/**
  @brief   Find the midpoint and edges of a cut through a slice.
  @param   aDataVec    cpl_vector with the data
  @param   aFrac       fraction of median used to identify an edge
  @param   aLeft       return left edge found
  @param   aRight      return right edge found
  @param   aHighSN     return if the input vector has S/N high enough for
                       significant edge detection
  @param   aIFU        the IFU number
  @return  the location of the derived midpoint as double,
           a negative value on error

  Go along the input data vector to search for the two outer edges of the
  slice. An edge is detected by comparing the data value to the median value
  along the data vector; if the difference is larger than aFrac times the
  median of the data vector, an edge was found. The returned midpoint is the
  arithmetic mean of the two edge positions for each slice.

  @qa Will be done in the calling function muse_trace().

  @error{return -1, a right edge could not be found}
  @error{return -2, a left edge could not be found}
  @error{set CPL_ERROR_ILLEGAL_INPUT\, return -3,
         the vector has too few elements}
  @error{set CPL_ERROR_ILLEGAL_INPUT\, return -4, aFrac is invalid (<0 or >1)}
  @error{set CPL_ERROR_NULL_INPUT\, return -5,
         the out parameters aLeft and aRight are NULL}
  @error{output error message\, return -11,
         faulty interpolation at right-hand edge}
  @error{output error message\, return -12,
         faulty interpolation at left-hand edge}
 */
/*---------------------------------------------------------------------------*/
double
muse_trace_edgefinder(const cpl_vector *aDataVec, double aFrac,
                      double *aLeft, double *aRight, cpl_boolean *aHighSN,
                      const unsigned char aIFU)
{
  int size = cpl_vector_get_size(aDataVec);
  /* stuff like i-1 doesn't work if the input vector is too small */
  cpl_ensure(size > 5, CPL_ERROR_ILLEGAL_INPUT, -3);
  cpl_ensure(aFrac > 0. && aFrac < 1., CPL_ERROR_ILLEGAL_INPUT, -4);
  cpl_ensure(aLeft && aRight, CPL_ERROR_NULL_INPUT, -5);

  /* compute the detection limit from the median and median deviation *
   * over the one-dimensional cut across the slices                   */
  double median = cpl_vector_get_median_const(aDataVec),
         mdev = muse_cplvector_get_adev_const(aDataVec, median),
         mean = cpl_vector_get_mean(aDataVec),
         stdev = cpl_vector_get_stdev(aDataVec),
         detlimit = aFrac * median;
  /* the result will be significant, if the flux at this part of *
   * the slice is larger than the noise; this is used to decide  *
   * whether to output a warning message in case of failure      */
  cpl_boolean significant = median > mdev && mean > stdev;
  if (aHighSN) {
    *aHighSN = significant;
  }
#if 0
  cpl_msg_debug(__func__, "median=%f+/-%f, mean=%f+/-%f, aFrac=%f --> edge "
                "detection limit=%f (%ssignificant), IFU %hhu", median, mdev,
                mean, stdev, aFrac, detlimit, significant ? "" : "NOT ", aIFU);
#endif

  /* preset edge positions and return arguments for error cases */
  *aRight = 0;
  *aLeft = 0;
  /* buffer access to vector */
  const double *ydata = cpl_vector_get_data_const(aDataVec);

  /* run the followingn at least once, iterate if the width is too small */
  int offset = 0; /* do the first iteration starting at the center *
                   * of the extracted buffer                       */
  do {
#if 0
    if (offset > 0) {
      cpl_msg_debug(__func__, "Iterating edge search with offset = %d pix (IFU "
                    "%hhu)", offset, aIFU);
    }
#endif

    /* start at the center and search outwards for the right-hand edge */
    int i;
    for (i = size/2 + offset; i < size; i++) {
      if (ydata[i] < detlimit) {
        /* i is the element where we fell below the limit. Use this to   *
         * determine the fractional position where the limit was reached *
         * using linear interpolation                                    */
        *aRight = i-1 + (detlimit - ydata[i-1]) / (ydata[i] - ydata[i-1]);
#if 0
        cpl_msg_debug(__func__, "r: %d..._%d_, %f/_%f_ ===> %f", i-1, i,
                      ydata[i-1], ydata[i], *aRight);
#endif
        /* if the found interpolation point is more than 1 pix away *
         * from the current pixel then the interpolation was wrong! */
        if (fabs(*aRight - i) > 1.) {
          /* Only display message and return the fault, if we actually got a few *
           * pixels away from the position where we started. Otherwise it will   *
           * just be a dark pixel near the slice center, that we can ignore.     *
           * This case would be a real problem and should actually not happen!   */
          if (significant && (i - (size/2 + offset)) > 2) {
            cpl_msg_debug(__func__, "Faulty interpolation of right-hand edge in "
                          "IFU %hhu: i=%d (start %d), *aRight=%f (%f..%f > %f > "
                          "%f)", aIFU, i, size/2 + offset, *aRight, ydata[i-2],
                          ydata[i-1], detlimit, ydata[i]);
            return -11;
          } /* if real problem */
          continue;
        } /* if large offset */
        break;
      } /* if data below limit */
    } /* for i (right-hand part of slice) */
    /* we arrived at upper limit of vector, so we didn't find the right edge! */
    if (i == size) {
      return -1;
    }

    /* start again at the center and search outwards for the left-hand edge */
    for (i = size/2 - offset; i >= 0; i--) {
      if (ydata[i] < detlimit) {
        /* again, use linear interpolation to find the exact position */
        *aLeft = i + (detlimit - ydata[i]) / (ydata[i+1] - ydata[i]);
#if 0
        cpl_msg_debug(__func__, "l: %d..._%d_, %f/_%f_ ===> %f", i+1, i,
                      ydata[i+1], ydata[i], *aLeft);
#endif
        if (fabs(*aLeft - i) > 1.) {
          if (significant && (size/2 - offset - i) > 2) {
            /* this would be a real problem, should never happen! */
            cpl_msg_debug(__func__, "Faulty interpolation of left-hand edge in "
                          "IFU %hhu: i=%d (start %d), *aLeft=%f (%f < %f < %f..%f"
                          ")", aIFU, i, size/2 - offset, *aLeft, ydata[i],
                          detlimit, ydata[i+1], ydata[i+2]);
            return -12;
          } /* if real problem */
          continue;
        } /* if large offset */
        break;
      } /* if data below limit */
    } /* for i (left-hand part of slice) */
    /* we arrived at lower limit of vector, so we didn't find the left edge! */
    if (i == -1) {
      return -2;
    }

    /* If we need to iterate, do so with an offset from the center that is 1  *
     * pixel larger than the smaller of the two distances that we found last. */
    int offsetold = offset;
    if ((size/2 - *aLeft) > (*aRight - size/2)) {
      offset = *aRight - size/2 + 2;
    } else {
      offset = size/2 - *aLeft + 2;
    }
    if (offset <= offsetold) {
      /* Ensure that the next iteration starts at least one    *
       * pixel further, otherwise we might not stop iterating! */
      offset++;
    }
    if (offset > size/2) {
      /* if the offset is already half the extent of the buffer, *
       * we won't find the edge(s) any more...                   */
      break;
    }
  } while ((*aRight - *aLeft + 1) < kMuseSliceLoLikelyWidth || (offset > size/2));

#if 0
  cpl_msg_debug(__func__, "result in IFU %hhu: %f %f --> %f", aIFU, *aLeft,
                *aRight, (*aLeft + *aRight)/2.);
#endif
  return (*aLeft + *aRight)/2.;
} /* muse_trace_edgefinder() */

/*---------------------------------------------------------------------------*/
/**
  @brief   Find more exact midpoint and edge positions using a difference
           vector of the input data.
  @param   aDiffVec   cpl_vector with the data
  @param   aLeft      initial value and return parameter for left edge
  @param   aRight     initial value and return parameter for right edge
  @param   aOffset    offset of coordinates to master flat image
  @param   aY         approximate center in the vertical direction
                      (just for debugging output)
  @param   aSlice     the slice number
  @param   aIFU       the IFU number
  @return  the location of the derived midpoint as double,
           a negative value on error

  Fit Gaussians to positive peak near initial position *aLeft and to negative
  dip near initial position *aRight to get more exact slice edges.  The
  returned midpoint is the arithmetic mean of the two edge positions for each
  slice.

  @error{return -1,
         output midpoint is shifted too much with regard to initial values}
  @error{set CPL_ERROR_ILLEGAL_INPUT\, return -3,
         the vector has too few elements}
  @error{set CPL_ERROR_NULL_INPUT\, return -5,
         the out parameters aLeft and aRight are NULL}
  @error{set CPL_ERROR_NULL_INPUT\, return -6,
         aLeft and aRight contain bad initial values}
 */
/*---------------------------------------------------------------------------*/
static double
muse_trace_refine_edge(cpl_vector *aDiffVec, double *aLeft, double *aRight,
                       int aOffset, double aY, const unsigned short aSlice,
                       const unsigned char aIFU)
{
#define TRACE_REFINE_MAX_SHIFT 0.25 /* more than 1/4 pix shift is too much */
#define TRACE_REFINE_RANGE 5
  int size = cpl_vector_get_size(aDiffVec);
  cpl_ensure(size > 5, CPL_ERROR_ILLEGAL_INPUT, -3);
  cpl_ensure(aLeft && aRight, CPL_ERROR_NULL_INPUT, -5);
  cpl_ensure(*aLeft > 0 && *aLeft < size &&
             *aRight > 0 && *aRight < size && *aRight > *aLeft,
             CPL_ERROR_ILLEGAL_INPUT, -6);

  /* keep initial values around */
  double left = *aLeft,
         right = *aRight,
         mid = (*aLeft + *aRight) / 2.;

  /* create two vectors for the actual fitting of both edges, *
   * plus two vectors that contain coordinates for the fit    */
  int nel = 2 * TRACE_REFINE_RANGE + 1; /* number of elements in vectors to fit */
  cpl_vector *vl = cpl_vector_new(nel), /* diff-data vectors */
             *vr = cpl_vector_new(nel),
             *pl = cpl_vector_new(nel), /* position vectors */
             *pr = cpl_vector_new(nel);
  /* extraction offsets so that the peaks are in the middle of the vectors */
  int loffset = (int)(left + 0.5)- TRACE_REFINE_RANGE + 1,
      roffset = (int)(right + 0.5) - TRACE_REFINE_RANGE + 1;
#if 0
  cpl_msg_debug(__func__, "input: %f/%f -> %d/%d",
                left, right, loffset, roffset);
#endif
  double *diff = cpl_vector_get_data(aDiffVec);
  int i;
  for (i = 0; i < nel; i++ ) {
    double d = diff[i + loffset - 1];
#if 0
    cpl_msg_debug(__func__, "l i=%d / %d: %f", i, i + loffset, d);
#endif
    cpl_vector_set(pl, i, i + loffset - 1);
    /* cut off negative values */
    cpl_vector_set(vl, i, d > 0 ? d : 0);
  } /* for i (around left start value) */
  for (i = 0; i < nel; i++ ) {
    /* need the inverse value for the right-hand edge */
    double d = -diff[i + roffset - 1];
#if 0
    cpl_msg_debug(__func__, "r i=%d / %d: %f", i, i + roffset, d);
#endif
    cpl_vector_set(pr, i, i + roffset - 1);
    /* cut off negative values */
    cpl_vector_set(vr, i, d > 0 ? d : 0);
  } /* for i (around right start value) */

  /* get state to possibly reset it */
  cpl_errorstate state = cpl_errorstate_get();
  /* background level of zero is a very good estimate on this *
   * difference data, really no need to fit that parameter    */
  double center, sigma, area, bglevel = 0, mse;
  cpl_fit_mode fitmode = CPL_FIT_CENTROID | CPL_FIT_STDEV | CPL_FIT_AREA;
  cpl_error_code rc1 = cpl_vector_fit_gaussian(pl, NULL, vl, NULL, fitmode,
                                               &center, &sigma, &area, &bglevel,
                                               &mse, NULL, NULL);
  /* center needs to be corrected for shift introduced by shift-diff procedure */
  center -= 0.5;
  if (rc1 == CPL_ERROR_CONTINUE || rc1 == CPL_ERROR_SINGULAR_MATRIX) {
    /* fit returned error, but parameters are supposed to be valid; use them, *
     * if they don't differ too much from the input, and reset the error      */
    if (fabs(center - *aLeft) < TRACE_REFINE_MAX_SHIFT) {
      *aLeft = center;
    }
  }
  if (rc1 != CPL_ERROR_NONE) {
    /* fit failed, keep the original value */
    cpl_errorstate_set(state);
  } else { /* successful, use the center of the fit */
    *aLeft = center;
  }
#if 0
  cpl_msg_debug(__func__, "fit l: %f %f %f (%f)", center, sigma, area, sqrt(mse));
#endif
  cpl_error_code rc2 = cpl_vector_fit_gaussian(pr, NULL, vr, NULL, fitmode,
                                               &center, &sigma, &area, &bglevel,
                                               &mse, NULL, NULL);
  center -= 0.5;
  if (rc2 == CPL_ERROR_CONTINUE || rc2 == CPL_ERROR_SINGULAR_MATRIX) {
    /* fit returned error, but parameters are supposed to be valid; use them, *
     * if they don't differ too much from the input, and reset the error      */
    if (fabs(center - *aRight) < TRACE_REFINE_MAX_SHIFT) {
      *aRight = center;
    }
  }
  if (rc2 != CPL_ERROR_NONE) {
    /* fit failed, keep the original value */
    cpl_errorstate_set(state);
  } else { /* successful, use the center of the fit */
    *aRight = center;
  }
#if 0
  cpl_msg_debug(__func__, "fit r: %f %f %f (%f)", center, sigma, area, sqrt(mse));
#endif

#if 0
  cpl_vector_dump(aDiffVec, stdout);
  printf("left\n");
  cpl_bivector *biv = cpl_bivector_wrap_vectors(pl, vl);
  cpl_bivector_dump(biv, stdout);
  cpl_bivector_unwrap_vectors(biv);
  printf("right\n");
  biv = cpl_bivector_wrap_vectors(pr, vr);
  cpl_bivector_dump(biv, stdout);
  cpl_bivector_unwrap_vectors(biv);
  fflush(stdout);
#endif
  cpl_vector_delete(vl);
  cpl_vector_delete(vr);
  cpl_vector_delete(pl);
  cpl_vector_delete(pr);

  double midpoint = (*aLeft + *aRight)/2.;
#if 0
  cpl_msg_debug(__func__, "refine: %f %f %f  %f %f %f",
                *aLeft, midpoint, *aRight,
                left - *aLeft, mid - midpoint, right - *aRight);
#endif
  /* skip points where measured centers don't agree; one could also *
   * check the edges but for those the shifts change with edgefrac  */
  if (fabs(mid - midpoint) > TRACE_REFINE_MAX_SHIFT) {
    cpl_msg_debug(__func__, "large refined shift around y=%.1f in slice %hu of "
                  "IFU %hhu: %f %f %f (%f %f %f) trace point will not be used",
                  aY, aSlice, aIFU, left + aOffset, midpoint + aOffset,
                  right + aOffset, left - *aLeft, mid - midpoint, right - *aRight);
    midpoint = -1.; /* make it fail */
  } else { /* success, add the offset */
    midpoint += aOffset;
  }
  /* add the offset to left and right in any case */
  *aLeft += aOffset;
  *aRight += aOffset;
  return midpoint;
} /* muse_trace_refine_edge() */

/*---------------------------------------------------------------------------*/
/**
  @brief   iterate the tracing solution to remove outliers
  @param   aX          matrix with the tracing x coordinates
  @param   aY          vector with the tracing y coordinates
  @param   aWidths     vector with the slice width at each point
  @param   aSlice      the number of the current slice (for debugging output)
  @param   aIFU        the IFU number
  @param   aFitorder   the polynomial order to use for the fit
  @param   aWSigma     rejection sigma to use for widths
  @param   aRSigma     rejection sigma to use for residuals
  @param   aMSE        output mean squared error of fit
                       (cpl_vector of MUSE_TRACE_NPOLY values)
  @return  The polynomial fits in a MUSE_TRACE_NPOLY-element array or NULL on
           error.

  This function expects at least 3 trace points in the input structures and
  will return CPL_ERROR_ILLEGAL_INPUT if this is not the case.

  aX, aY, and aWidth will be changed by this function, if outliers are present
  they are removed.

  The sigmas are given in terms of median absolute deviations.

  The aMSE values will have a value of FLT_MAX in case the polynomial fit
  returned an error.

  The returned array of polynomials has to be freed using
  muse_trace_polys_delete() after use.
 */
/*---------------------------------------------------------------------------*/
static cpl_polynomial **
muse_trace_iterate_fit(cpl_matrix *aX, cpl_vector *aY, cpl_vector *aWidths,
                       const unsigned short aSlice, const unsigned char aIFU,
                       const unsigned int aFitorder, const float aWSigma,
                       const float aRSigma, cpl_vector *aMSE)
{
  cpl_ensure(cpl_vector_get_size(aWidths) >= 3, CPL_ERROR_ILLEGAL_INPUT, NULL);
  /* not a critical argument but silences the build warning without debug output */
  cpl_ensure(aSlice >= 1 && aSlice <= kMuseSlicesPerCCD,
             CPL_ERROR_ILLEGAL_INPUT, NULL);

  /* throw away trace points with wildly different widths */
  double wmean = cpl_vector_get_mean(aWidths),
         wmedian = cpl_vector_get_median_const(aWidths),
         wstdev = cpl_vector_get_stdev(aWidths),
         wmdev = muse_cplvector_get_adev_const(aWidths, wmedian);
#if 0
  cpl_msg_debug(__func__, "width (1st): mean %6.3f +/- %5.3f, median %6.3f +/- "
                "%5.3f (%"CPL_SIZE_FORMAT" points)", wmean, wstdev, wmedian, wmdev,
                cpl_vector_get_size(aWidths));
#endif
  if ((wmean - wstdev < kMuseSliceLoLikelyWidth ||
       wmedian - wmdev < kMuseSliceLoLikelyWidth) &&
      (wmean < kMuseSliceLoLikelyWidth || wmedian < kMuseSliceLoLikelyWidth)) {
    cpl_msg_debug(__func__, "slice %hu of IFU %hhu seems to be very narrow "
                  "initially (widths: mean %6.3f +/- %5.3f, median %6.3f +/- "
                  "%5.3f)!", aSlice, aIFU, wmean, wstdev, wmedian, wmdev);
  }
  if ((wmean + wstdev > kMuseSliceHiLikelyWidth ||
       wmedian + wmdev > kMuseSliceHiLikelyWidth) &&
      (wmean > kMuseSliceHiLikelyWidth || wmedian > kMuseSliceHiLikelyWidth)) {
    cpl_msg_debug(__func__, "slice %hu of IFU %hhu seems to be very wide "
                  "initially (widths: mean %6.3f +/- %5.3f, median %6.3f +/- "
                  "%5.3f)!", aSlice, aIFU, wmean, wstdev, wmedian, wmdev);
  }

  /* loop through all trace points and remove grossly deviant points *
   * where the slice is far too narrow or far too wide to be real    */
  int i;
  for (i = 0; i < cpl_vector_get_size(aWidths); i++) {
    double width = cpl_vector_get(aWidths, i);
    if (width > kMuseSliceLoLikelyWidth && width < kMuseSliceHiLikelyWidth) {
      /* good widths */
      continue;
    }
    /* guard against removing the last element */
    if (cpl_vector_get_size(aWidths) == 1) {
      cpl_msg_warning(__func__, "trying to remove the last vector/matrix "
                      "element in slice %hu of IFU %hhu when checking widths",
                      aSlice, aIFU);
      break;
    }
    /* bad width, remove element */
    cpl_matrix_erase_columns(aX, i, 1);
    muse_cplvector_erase_element(aY, i);
    muse_cplvector_erase_element(aWidths, i);
    i--; /* we stay at this position to see what moved here */
  } /* for i */

  wmean = cpl_vector_get_mean(aWidths);
  wmedian = cpl_vector_get_median_const(aWidths);
  wstdev = cpl_vector_get_stdev(aWidths);
  wmdev = muse_cplvector_get_adev_const(aWidths, wmedian);
#if 0
  cpl_msg_debug(__func__, "width (2nd): mean %6.3f+/-%5.3f, median %6.3f+/-%5.3f (%d points)",
                wmean, wstdev, wmedian, wmdev, cpl_vector_get_size(aWidths));
#endif
  if ((wmean - wstdev < kMuseSliceLoLikelyWidth ||
       wmedian - wmdev < kMuseSliceLoLikelyWidth) &&
      (wmean < kMuseSliceLoLikelyWidth || wmedian < kMuseSliceLoLikelyWidth)) {
    cpl_msg_warning(__func__, "slice %hu of IFU %hhu seems to be very narrow "
                    "after iteration (widths: mean %6.3f +/- %5.3f, median %6.3f"
                    " +/- %5.3f)!", aSlice, aIFU, wmean, wstdev, wmedian, wmdev);
  }
  if ((wmean + wstdev > kMuseSliceHiLikelyWidth ||
       wmedian + wmdev > kMuseSliceHiLikelyWidth) &&
      (wmean > kMuseSliceHiLikelyWidth || wmedian > kMuseSliceHiLikelyWidth)) {
    cpl_msg_warning(__func__, "slice %hu of IFU %hhu seems to be very wide "
                    "after iteration (widths: mean %6.3f +/- %5.3f, median %6.3f"
                    " +/- %5.3f)!", aSlice, aIFU, wmean, wstdev, wmedian, wmdev);
  }

  /* again loop through all trace points to now reject based on sigma */
  for (i = 0; i < cpl_vector_get_size(aWidths); i++) {
    double width = cpl_vector_get(aWidths, i);
#if 0
    cpl_msg_debug(__func__, "i=%d: %f <? %f <? %f", i,
                  wmedian - aWSigma * wmdev, width, wmedian + aWSigma * wmdev);
#endif
    if (width > (wmedian - aWSigma * wmdev) &&
        width < (wmedian + aWSigma * wmdev)) {
      /* good widths */
      continue;
    }
    /* guard against removing the last element */
    if (cpl_vector_get_size(aWidths) == 1) {
      cpl_msg_warning(__func__, "trying to remove the last vector/matrix "
                      "element in slice %hu of IFU %hhu when checking fit "
                      "sigma", aSlice, aIFU);
      break;
    }
    /* bad width, remove element */
    cpl_matrix_erase_columns(aX, i, 1);
    muse_cplvector_erase_element(aY, i);
    muse_cplvector_erase_element(aWidths, i);
    i--; /* we stay at this position to see what moved here */
  } /* for i */

  /* create table column from vector */
  cpl_table *wtable = cpl_table_new(cpl_vector_get_size(aWidths));
  cpl_table_new_column(wtable, "widths", CPL_TYPE_DOUBLE);
  memcpy(cpl_table_get_data_double(wtable, "widths"),
         cpl_vector_get_data(aWidths), cpl_vector_get_size(aWidths));
  double mse, chisq;
  cpl_polynomial *tracefit = muse_utils_iterate_fit_polynomial(aX, aY, NULL, wtable,
                                                               aFitorder, aRSigma,
                                                               &mse, &chisq);
  /* adapt length of input vector, copy output widths back into it */
  cpl_vector_set_size(aWidths, cpl_vector_get_size(aY));
  memcpy(cpl_vector_get_data(aWidths), cpl_table_get_data_double(wtable, "widths"),
         cpl_vector_get_size(aWidths));
  cpl_table_delete(wtable);
  if (!tracefit) { /* if the fit didn't work then something went wrong */
    cpl_vector_fill(aMSE, FLT_MAX);
    return NULL;
  }
  /* also save in input vector */
  cpl_vector_set(aMSE, MUSE_TRACE_CENTER, mse);

  char *dodebug = getenv("MUSE_DEBUG_TRACE");
  if (dodebug && atoi(dodebug) > 0) {
    printf("Polynomial trace fit for slice %hu of IFU %hhu (mse=%g, "
           "chi**2=%g):\n", aSlice, aIFU, mse, chisq);
    cpl_polynomial_dump(tracefit, stdout);
    fflush(stdout);
  }

  /* Now we have a polynomial defining the center and the array of widths. *
   * Convert this into separate matrices for left and right edges and redo *
   * the polynomial fits for both edges.                                   */
  cpl_vector *edge[MUSE_TRACE_NPOLY - 1] = {
    cpl_vector_new(cpl_vector_get_size(aY)),
    cpl_vector_new(cpl_vector_get_size(aY))
  };
  for (i = 0; i < cpl_vector_get_size(aWidths); i++) {
    double x = cpl_vector_get(aY, i),
           halfwidth = cpl_vector_get(aWidths, i) / 2.;
#if 0
    cpl_msg_debug(__func__, "x=%f (%f...%f)", x, x - halfwidth, x + halfwidth);
#endif
    cpl_vector_set(edge[MUSE_TRACE_LEFT - 1], i, x - halfwidth);
    cpl_vector_set(edge[MUSE_TRACE_RIGHT - 1], i, x + halfwidth);
  } /* for i */
#if 0
  printf("left:\n");
  cpl_vector_dump(edge[0], stdout);
  printf("right:\n");
  cpl_vector_dump(edge[1], stdout);
  fflush(stdout);
#endif

  cpl_polynomial **fit = cpl_calloc(MUSE_TRACE_NPOLY, sizeof(cpl_polynomial *));
  fit[MUSE_TRACE_CENTER] = tracefit;

  int ipoly;
  for (ipoly = 1; ipoly < MUSE_TRACE_NPOLY; ipoly++) { /* from 1, skip center */
    /* fit again, but do not iterate, i.e. use a very high rejection sigma */
    fit[ipoly] = muse_utils_iterate_fit_polynomial(aX, edge[ipoly - 1], NULL,
                                                   NULL, aFitorder, FLT_MAX,
                                                   &mse, &chisq);
    cpl_vector_set(aMSE, ipoly, mse);
    /* we are now done with the extra fit input data, too */
    cpl_vector_delete(edge[ipoly - 1]);
  } /* for ipoly */

#if 0
  printf("resulting polynomials (center, left, and right):\n");
  cpl_polynomial_dump(fit[MUSE_TRACE_CENTER], stdout);
  cpl_polynomial_dump(fit[MUSE_TRACE_LEFT], stdout);
  cpl_polynomial_dump(fit[MUSE_TRACE_RIGHT], stdout);
  printf("MSEs:\n");
  cpl_vector_dump(aMSE, stdout);
  fflush(stdout);
#endif

  return fit;
} /* muse_trace_iterate_fit() */

/*----------------------------------------------------------------------------*/
/**
 * @brief MUSE tracing sample points table definition.
 */
/*----------------------------------------------------------------------------*/
const muse_cpltable_def muse_tracesamples_def[] = {
  { "slice", CPL_TYPE_INT, "", "%02d", "slice number", CPL_TRUE},
  { "y", CPL_TYPE_FLOAT, "pix", "%6.1f", "y position on CCD", CPL_TRUE},
  { "mid", CPL_TYPE_FLOAT, "pix", "%8.3f",
    "midpoint of the slice at this y position", CPL_TRUE},
  { "left", CPL_TYPE_FLOAT, "pix", "%8.3f",
    "left edge of the slice at this y position", CPL_TRUE},
  { "right", CPL_TYPE_FLOAT, "pix", "%8.3f",
    "right edge of the slice at this y position", CPL_TRUE},
  { NULL, 0, NULL, NULL, NULL, CPL_FALSE }
};

/*---------------------------------------------------------------------------*/
/**
  @brief   carry out the tracing of the slices on CCD, save parameters in table
  @param   aImage      muse_image that holds the image for tracing
  @param   aNSum       number of image rows to combine when tracing;
                       -1<=aNSum<=1: no combination
                       < -1: use average, > 1: use median
  @param   aEdgeFrac   fraction of median used to identify an edge
  @param   aFitorder   polynomial order used for tracing
  @param   aSamples    this table is filled with debug information on
                       tracing sample points
  @return  the cpl_table* containing the trace parameters, or NULL on error

  The algorithms used here do not work well to take into account bad pixels.
  The dq component of aImage is therefore used to interpolate all bad pixels
  as a first step. (Note however, that the original image is not changed.)
  Then create a horizontal cut through the slices. For this purpose, cut
  out a number of rows (on the order of 100 pix high) of the input image near
  its vertical center, use a median filter to remove cosmic rays, and average
  all rows to form a 1D cut. Using this cut, determine the center of each
  slice in the vertical center of the CCD, using muse_trace_locate_slices(),
  taking into account the aEdgeFrac parameter.
  Loop over all slices and within each slice over all tracepoints (determined
  by aNSum). For each tracepoint, create a cut by collapsing aNSum rows over
  a width larger than the expected slice width. For this cut, determine the
  exact center of the slice, using muse_trace_edgefinder(), taking into
  account the aEdgeFrac parameter.
  Fit the trace with a polynomial of order aFitorder and store the polynomial
  coefficients in the output table.

  @qa Statistics on the edge positions within each slice and comparing left and
      right edge of the slices should give a robust indication of tracing
      failures.

  @error{set CPL_ERROR_NULL_INPUT\, return NULL, input image is missing}
  @error{set CPL_ERROR_ILLEGAL_INPUT\, return NULL, aNSum is not positive}
  @error{set CPL_ERROR_ILLEGAL_INPUT\, return NULL,
         aEdgeFrac is invalid (<=0 or >=1)}
  @error{set CPL_ERROR_ILLEGAL_INPUT\, return NULL, aFitOrder is not positive}
  @error{return NULL\, propagating the error set by muse_trace_locate_slices,
         could not carry out first guess of slice positions}
  @error{return NULL\, propagate error of cpl_table_new,
         could not allocate space for tracing table}
  @error{output warning\, single failures can occur in spectral regions of low S/N,
         edge finding failed at one position}
 */
/*---------------------------------------------------------------------------*/
cpl_table *
muse_trace(const muse_image *aImage, int aNSum, double aEdgeFrac, int aFitorder,
           cpl_table **aSamples)
{
  cpl_ensure(aImage && aImage->data, CPL_ERROR_NULL_INPUT, NULL);
  cpl_ensure(aNSum > 0 && aEdgeFrac > 0. && aEdgeFrac < 1. && aFitorder > 0,
             CPL_ERROR_ILLEGAL_INPUT, NULL);

  /* count number of tracepoints */
  int ny  = cpl_image_get_size_y(aImage->data),
      npoints = (ny - 1) / aNSum;
  unsigned short nsearchslices = kMuseSlicesPerCCD;
  cpl_boolean slice_number_hack = getenv("MUSE_AIT_HACK_SLICE_NUMBER")
                                && atoi(getenv("MUSE_AIT_HACK_SLICE_NUMBER")) > 0
                                && atoi(getenv("MUSE_AIT_HACK_SLICE_NUMBER")) < 49;
  unsigned char ifu = muse_utils_get_ifu(aImage->header);
  if (slice_number_hack) {
    nsearchslices = atoi(getenv("MUSE_AIT_HACK_SLICE_NUMBER"));
    cpl_msg_warning(__func__, "Overriding number of slices to search in IFU "
                    "%hhu to %hu!", ifu, nsearchslices);
  }
  cpl_msg_info(__func__, "Working with %hu slices, %d image rows, and %d "
               "tracepoints in IFU %hhu", nsearchslices, ny, npoints, ifu);

  /* check if this is IFU 24 and data taken since it was installed at Paranal *
   * in January 2014, because that has the strange vignetting in channel 24   */
  cpl_boolean chan24paranal = (ifu == 24);
  const char *dateobs = muse_pfits_get_dateobs(aImage->header);
  if (chan24paranal && dateobs) { /* it is IFU 24 and we do have a DATE-OBS */
    chan24paranal = atoi(dateobs) >= 2014;
  }
  if (chan24paranal) {
    cpl_msg_info(__func__, "Using overrides for IFU 24 since 2014: due to field"
                 " vignetting, left-hand edges (as seen on the CCD) of slices "
                 "37 to 48 may be only approximate!");
  }

  /* duplicate input image to be able to interpolate bad pixels */
  muse_image *image = muse_image_new();
  image->data = cpl_image_duplicate(aImage->data);
  if (aImage->dq) {
    image->dq = cpl_image_duplicate(aImage->dq);
  } else {
    image->dq = cpl_image_new(cpl_image_get_size_x(aImage->data), ny, CPL_TYPE_INT);
  }
  if (aImage->header) { /* header is used by muse_trace_horizontal_cut() */
    image->header = cpl_propertylist_duplicate(aImage->header);
  }
  /* stat and header are not needed here */
  muse_image_reject_from_dq(image);
  cpl_detector_interpolate_rejected(image->data);

  /* get starting guesses for the midpoint of each slice */
#define NROWCOLLAPSE 15
  cpl_vector *cut = muse_trace_horizontal_cut(image, NROWCOLLAPSE);
  cpl_vector *centers = muse_trace_locate_slices(cut, nsearchslices, aEdgeFrac,
                                                 ifu);
  cpl_vector_delete(cut);
  if (!centers) {
    cpl_msg_error(__func__, "Could not carry out first guess of slice positions "
                  "in IFU %hhu!", ifu);
    muse_image_delete(image);
    return NULL;
  }

  /* create the output table */
  cpl_table *tracetable = cpl_table_new(kMuseSlicesPerCCD);
  if (!tracetable) {
    cpl_msg_error(__func__, "Could not create output trace table for IFU %hhu: "
                  "%s", ifu, cpl_error_get_message());
    muse_image_delete(image);
    return NULL;
  }

  /* prepare output table */
  cpl_table_new_column(tracetable, MUSE_TRACE_TABLE_COL_SLICE_NO, CPL_TYPE_INT);
  cpl_table_set_column_unit(tracetable, MUSE_TRACE_TABLE_COL_SLICE_NO, "No");
  cpl_table_set_column_format(tracetable, MUSE_TRACE_TABLE_COL_SLICE_NO, "%2d");
  cpl_table_new_column(tracetable, MUSE_TRACE_TABLE_COL_WIDTH, CPL_TYPE_FLOAT);
  cpl_table_set_column_unit(tracetable, MUSE_TRACE_TABLE_COL_WIDTH, "pix");
  cpl_table_set_column_format(tracetable, MUSE_TRACE_TABLE_COL_WIDTH, "%6.3f");
  int ipoly;
  for (ipoly = 0; ipoly < MUSE_TRACE_NPOLY; ipoly++) {
    char *colname;
    int j;
    for (j = 0; j <= aFitorder; j++) {
      /* create column name, start coefficient names at 0 */
      colname = cpl_sprintf(MUSE_TRACE_TABLE_COL_COEFF, ipoly, j);
      cpl_table_new_column(tracetable, colname, CPL_TYPE_DOUBLE);
      /* fit coeff are in pixel space */
      cpl_table_set_column_unit(tracetable, colname, "pix");
      cpl_table_set_column_format(tracetable, colname, "%12.5e");
      cpl_free(colname);
    }
    colname = cpl_sprintf(MUSE_TRACE_TABLE_COL_MSE, ipoly);
    cpl_table_new_column(tracetable, colname, CPL_TYPE_DOUBLE);
    cpl_table_set_column_unit(tracetable, colname, "pix");
    cpl_table_set_column_format(tracetable, colname, "%12.5e");
    cpl_free(colname);
  } /* for ipoly */

  int isamplesrow = -1; /* track the current row in the samples table */
  if (aSamples) {
    /* create table of the sampled points for debugging */
    *aSamples = muse_cpltable_new(muse_tracesamples_def,
                                  kMuseSlicesPerCCD * npoints);
  }

  /* create some kind of image derivative by subtracting a 1 pix *
   * shifted version of the input image from the input image     */
  cpl_image *shiftdiff = cpl_image_duplicate(image->data);
  cpl_image_shift(shiftdiff, 1, 0);
  /* subtract from original, store in new one */
  cpl_image_multiply_scalar(shiftdiff, -1);
  cpl_image_add(shiftdiff, image->data);

  /* islice loops over all slices, j over all rows, k over all tracepoints */
  unsigned short islice;
  for (islice = 0; islice < kMuseSlicesPerCCD; islice++) {
    int nfailed = 0; /* count the number of failed tracepoints in this slice */
    cpl_matrix *xtrace = cpl_matrix_new(1, npoints);
    cpl_vector *ytrace = cpl_vector_new(npoints);
    /* track slice width for statistics */
    cpl_vector *widths = cpl_vector_new(npoints);

    /* loops over vertical points (j) and tracepoints (k), resp. */
    int j, k, knum; /* the latter is just a counter for dianostic output */
    for (j = 1, k = 0, knum = 1; j <= ny - aNSum; j += aNSum, k++, knum++) {
      /* extract slice section and collapse to 1d cut             */
      /* (see muse_tracing.h for the definition of TRACE_BINSIZE) */
      int noffset = (int)cpl_vector_get(centers, islice) - TRACE_BINSIZE,
          ilo = noffset,
          ihi = (int)cpl_vector_get(centers, islice) + TRACE_BINSIZE,
          jlo = j,
          jhi = j + aNSum - 1;
#if 0
      cpl_msg_debug(__func__, "slice=%d, center=%f, cut region: %d,%d,%d,%d",
                    (int)islice + 1, cpl_vector_get(centers, islice),
                    ilo, ihi, jlo, jhi);
#endif
      cpl_image *tmp = cpl_image_collapse_window_create(image->data,
                                                        ilo, jlo, ihi, jhi,
                                                        0); /* collapse vertically */
      cpl_image_divide_scalar(tmp, aNSum);
      cut = cpl_vector_new_from_image_row(tmp, 1);
      cpl_image_delete(tmp);

      /* find midpoint and edges of the cut */
      double left, right;
      cpl_boolean highSN = CPL_TRUE;
      double midpoint = muse_trace_edgefinder(cut, aEdgeFrac, &left, &right,
                                              &highSN, ifu);
      cpl_vector_delete(cut);
#if 0 /* same output as above, but with the high S/N classification added */
      cpl_msg_debug(__func__, "slice=%d, center=%f, cut region: %d,%d,%d,%d, "
                    "%s S/N", (int)islice + 1, cpl_vector_get(centers, islice),
                    ilo, ihi, jlo, jhi, highSN ? "high" : "low");
#endif
      /* keep left edge of the standard edge detection, in      *
       * case we are dealing with the slices 37 to 48 in IFU 24 */
      double left1 = left + noffset;

      cpl_errorstate state = cpl_errorstate_get();
      /* refine edge positions again using difference image *
       * and Gaussian fits to the edges                     */
      tmp = cpl_image_collapse_window_create(shiftdiff, ilo, jlo, ihi, jhi,
                                             0); /* collapse along rows */
      cpl_image_divide_scalar(tmp, aNSum);
      cut = cpl_vector_new_from_image_row(tmp, 1);
      cpl_image_delete(tmp);
      midpoint = muse_trace_refine_edge(cut, &left, &right, noffset,
                                        (jlo + jhi) / 2., islice + 1, ifu);
      cpl_vector_delete(cut);

      /* slices 37 to 48 on the CCD have rounded left edge for IFU 24 */
      if (midpoint < 0 && midpoint > -2 && islice+1 >= 37 && chan24paranal) {
        cpl_msg_debug(__func__, "IFU24 problem? slice %d, y = %f: refined "
                      "%f < %f < %f", islice+1, ((double)jlo + jhi) / 2.,
                      left, midpoint, right);
        if (left1 < left) {
          /* If the edge detection was left of the Gaussian center, then that *
           * fit was affected by the rounded "edge" and needs to be reset.    */
          left = left1;
          cpl_msg_debug(__func__, "IFU24 problem! slice %d y = %f: corrected "
                        "%f < %f < %f", islice+1, ((double)jlo + jhi) / 2.,
                        left, (left + right) / 2., right);
        }
        /* the refined value for the right edge should still be fine, keep it */
        midpoint = (left + right) / 2.;
      } /* if slice 37-48 problem */

      if (midpoint > 0) { /* edge searching was successful */
        cpl_matrix_set(xtrace, 0, k, (jlo + jhi) / 2.);
        cpl_vector_set(ytrace, k, midpoint);
        cpl_vector_set(widths, k, right - left);

        if (aSamples) {
          /* save the points for plotting/debugging */
          if (++isamplesrow+1 > cpl_table_get_nrow(*aSamples)) {
            cpl_table_set_size(*aSamples, isamplesrow+1);
          }
          cpl_table_set_int(*aSamples, "slice", isamplesrow, islice + 1);
          cpl_table_set_float(*aSamples, "y", isamplesrow, (jlo + jhi) / 2.);
          cpl_table_set_float(*aSamples, "mid", isamplesrow, midpoint);
          cpl_table_set_float(*aSamples, "left", isamplesrow, left);
          cpl_table_set_float(*aSamples, "right", isamplesrow, right);
        }

        /* we were successful, so we can go to the next trace point */
        continue;
      }

      /* Error handling for the failure case follows. Only output warning, *
       * if many tracepoints (> 10%) are lost in the current slice. Only  *
       * count failures with significant flux levels (high S/N).           */
      cpl_errorstate_set(state);
      if (highSN) {
        nfailed++;
      }
#if 0
      cpl_msg_debug(__func__, "slice=%d, nfailed=%d, tracepoint=%d, midpoint="
                    "%f, y=%d", (int)islice + 1, nfailed, knum, midpoint, j);
#endif
      if (nfailed > 0.1*npoints && midpoint == -1.) {
        cpl_msg_warning(__func__, "failure %d in slice %d of IFU %hhu: lost "
                        "trace at y=%d (tracepoint %d of %d)", nfailed,
                        (int)islice + 1, ifu, j, knum, npoints);
      }

      /* resize vector and matrix, so that they don't include the failure */
      int oldsize = cpl_vector_get_size(ytrace); /* should all have same size */
      cpl_vector_set_size(ytrace, oldsize - 1);
      cpl_matrix_resize(xtrace, 0, 0, 0, -1);
      cpl_vector_set_size(widths, oldsize - 1);
      k--; /* set current matrix/vector index backwards */
    } /* for j */
#if 0
    /* compare number of tracepoints to points in vectors, *
     * 1 was already added to k (which started at 0)       */
    printf("k=%d tracepoints (should be equal to %d)\n", k, npoints);
    cpl_matrix_dump(xtrace, stdout), fflush(stdout);
    cpl_vector_dump(ytrace, stdout), fflush(stdout);
#endif

    /* iterate through input points and the fit with 5 sigma rejections */
    const float kWSigma = 5, kRSigma = 5;
    cpl_msg_debug(__func__, "Fitting traces in slice %d of IFU %hhu (kWSigma=%"
                  ".1f, kRSigma=%.1f)", (int)islice + 1, ifu, kWSigma, kRSigma);
    cpl_vector *mse = cpl_vector_new(MUSE_TRACE_NPOLY);
    cpl_vector_fill(mse, -1.);
    cpl_polynomial **tracefits = muse_trace_iterate_fit(xtrace, ytrace, widths,
                                                        islice + 1, ifu, aFitorder,
                                                        kWSigma, kRSigma, mse);
    /* use the final mean width for the trace solution */
    double wmean = cpl_vector_get_mean(widths);

    /* reset status after fitting procedures and clean up */
    cpl_matrix_delete(xtrace);
    cpl_vector_delete(ytrace);
    cpl_vector_delete(widths);

    if (!tracefits) {
      /* we need to print an error message and skip the table entries */
      cpl_msg_error(__func__, "The trace fit in slice %d of IFU %hhu failed",
                    (int)islice + 1, ifu);
      cpl_vector_delete(mse);
      continue;
    }

    /* row numbers start at 1 not 0 */
    cpl_table_set_int(tracetable, MUSE_TRACE_TABLE_COL_SLICE_NO, islice,
                      islice + 1);
    cpl_table_set_float(tracetable, MUSE_TRACE_TABLE_COL_WIDTH, islice, wmean);
    for (ipoly = 0; ipoly < MUSE_TRACE_NPOLY; ipoly++) {
      if (!tracefits[ipoly]) {
        cpl_msg_error(__func__, "The fit %d in slice %d of IFU %hhu failed",
                      ipoly, (int)islice + 1, ifu);
        continue;
      }
      char *colname = cpl_sprintf(MUSE_TRACE_TABLE_COL_MSE, ipoly);
      cpl_table_set_double(tracetable, colname, islice,
                           cpl_vector_get(mse, ipoly));
      cpl_free(colname);
      /* j loops over all orders of the polynomial */
      for (j = 0; j <= aFitorder; j++) {
        cpl_size pows[1] = { j }; /* trick to access the polynomial */

        colname = cpl_sprintf(MUSE_TRACE_TABLE_COL_COEFF, ipoly, j);
        cpl_errorstate prestate = cpl_errorstate_get();
        double coeff = cpl_polynomial_get_coeff(tracefits[ipoly], pows);
#define SLOPE_WARN_LIMIT 0.015
        if (j == 1 && fabs(coeff) > SLOPE_WARN_LIMIT) {
          cpl_msg_warning(__func__, "1st order coefficient of the %s tracing "
                          "polynomial is unexpectedly large in slice %d of IFU"
                          " %hhu: |%f| > %f", muse_trace_poly_strings[ipoly],
                          (int)islice + 1, ifu, coeff, SLOPE_WARN_LIMIT);
        }
        cpl_table_set_double(tracetable, colname, islice, coeff);
        if (!cpl_errorstate_is_equal(prestate)) {
          cpl_msg_warning(__func__, "Problem writing to field %s in trace table"
                          " for IFU %hhu: %s", colname, ifu,
                          cpl_error_get_message());
        }
        cpl_free(colname);
      } /* for j (all polynomial orders) */
    } /* for ipoly */
    cpl_vector_delete(mse);
    muse_trace_polys_delete(tracefits);
  } /* for islice */
  cpl_vector_delete(centers);
  cpl_image_delete(shiftdiff);
  if (aSamples) {
    /* cut samples table to the actually used number of rows */
    cpl_table_set_size(*aSamples, ++isamplesrow);
  }

  if (slice_number_hack) {
    cpl_msg_warning(__func__, "Will try to fix the slices in IFU %hhu, %"
                    CPL_SIZE_FORMAT" seem to be bad!", ifu,
                    cpl_table_count_invalid(tracetable, MUSE_TRACE_TABLE_COL_WIDTH));
    cpl_table_dump(tracetable, 0, 100, stdout); /* be sure to output _all_ slices */
    fflush(stdout);
    /* add fake slices for those that failed */
    for (islice = 0; islice < cpl_table_get_nrow(tracetable); islice++) {
      double width = cpl_table_get(tracetable, MUSE_TRACE_TABLE_COL_WIDTH, islice, NULL);
      if (width < 40.) {
        cpl_error_set_message(__func__, CPL_ERROR_ILLEGAL_OUTPUT, "slice %d of "
                              "IFU %hhu was narrow (%f), erased it", (int)islice + 1,
                              ifu, width);
        cpl_table_erase_window(tracetable, islice--, 1);
      } /* if */
    } /* for islice */
    double last = 0;
    for (islice = 0; islice < cpl_table_get_nrow(tracetable); islice++) {
      double cen = cpl_table_get(tracetable, "tc0_00", islice, NULL);
      if (cen - last > 100) { /* insert a slice */
        unsigned short iref = islice - 1;
        cpl_table *row = cpl_table_extract(tracetable, iref, 1);
        /* the slice numbering is fixed below */
        double offset = 84.5; /* typical offset */
        cpl_table_add_scalar(row, MUSE_TRACE_TABLE_COL_SLICE_NO, 1);
        cpl_table_add_scalar(row, "tc0_00", offset);
        cpl_table_add_scalar(row, "tc1_00", offset);
        cpl_table_add_scalar(row, "tc2_00", offset);
        cpl_table_add_scalar(row, "MSE0", 1.); /* signify by a large MSE that this is fake */
        cpl_table_add_scalar(row, "MSE1", 1.);
        cpl_table_add_scalar(row, "MSE2", 1.);
        cpl_table_insert(tracetable, row, islice);
#if 0
        printf("rowtable (islice=%hu):\n", islice);
        cpl_table_dump(row, 0, 100, stdout);
        fflush(stdout);
        printf("tracetable (islice=%hu):\n", islice);
        cpl_table_dump(tracetable, islice - 2, 10, stdout);
        fflush(stdout);
#endif
        cen = cpl_table_get(row, "tc0_00", 0, NULL); /* new center */
        cpl_table_delete(row);
        cpl_error_set_message(__func__, CPL_ERROR_ILLEGAL_OUTPUT, "slice was "
                              "missing before slice %d of IFU %hhu, copied "
                              "from slice %d", (int)islice + 1, ifu, (int)iref + 1);
      }
      last = cen;
    } /* for islice */
    for (islice = 0; islice < cpl_table_get_nrow(tracetable); islice++) {
      unsigned short sliceno = cpl_table_get_int(tracetable,
                                                 MUSE_TRACE_TABLE_COL_SLICE_NO,
                                                 islice, NULL);
      if (sliceno != islice + 1) {
        cpl_msg_warning(__func__, "Resetting entry at table row index %hu to "
                        "correct slice number in IFU %hhu (%d instead of %hu)",
                        islice, ifu, (int)islice + 1, sliceno);
        cpl_table_set_int(tracetable, MUSE_TRACE_TABLE_COL_SLICE_NO, islice,
                          islice + 1);
      }
    } /* for islice */
  } /* if slice_number_hack */

  /* give the user some useful output to read */
  cpl_msg_info(__func__, "Found %"CPL_SIZE_FORMAT" slices of width %4.1f+/-%3.1f"
               " pix (%4.1f pix...%4.1f pix) in IFU %hhu",
               cpl_table_get_nrow(tracetable),
               cpl_table_get_column_mean(tracetable, MUSE_TRACE_TABLE_COL_WIDTH),
               cpl_table_get_column_stdev(tracetable, MUSE_TRACE_TABLE_COL_WIDTH),
               cpl_table_get_column_min(tracetable, MUSE_TRACE_TABLE_COL_WIDTH),
               cpl_table_get_column_max(tracetable, MUSE_TRACE_TABLE_COL_WIDTH),
               ifu);
  muse_image_delete(image);

  return tracetable;
} /* muse_trace() */

/*---------------------------------------------------------------------------*/
/**
  @brief   determine order of tracing polynomial from table
  @param   aTable   trace table holding the trace solution
  @return  the polynomial order or a negative value on error

  Assumption: the table only contains the coefficients, the slice number,
  the MSE, and the slice width.
 */
/*---------------------------------------------------------------------------*/
int
muse_trace_table_get_order(const cpl_table *aTable)
{
  /* cpl_table_get_ncol() returns -1 on error so we   *
   * automatically get a negative value on error here */
  /* There are two extra values, the MSEs; the rest of the columns are the    *
   * coefficients of the MUSE_TRACE_NPOLY polynomials, including zeroth order */
  return (cpl_table_get_ncol(aTable) - 2) / MUSE_TRACE_NPOLY - 2;
} /* muse_trace_table_get_order() */

/*---------------------------------------------------------------------------*/
/**
  @brief   construct polynomial from the trace table entry for the given slice
  @param   aTable   trace table holding the trace solution
  @param   aSlice   the slice number, between 1 and kMuseSlicesPerCCD
  @return  the polynomials defining the trace of the given slice as a
           three-element array or NULL on error

  The returned polynomials have to be deallocated using
  muse_trace_polys_delete() after use.

  @error{set CPL_ERROR_NULL_INPUT\, return NULL, aTable is NULL}
  @error{set CPL_ERROR_ILLEGAL_INPUT\, return NULL,
         aSlice is outside the valid range of slices for MUSE}
  @error{set CPL_ERROR_DATA_NOT_FOUND\, return NULL,
         the requested slice cannot be found in the table}
  @error{set CPL_ERROR_ILLEGAL_OUTPUT\, return NULL,
         a coefficient in aTable cannot be read}
 */
/*---------------------------------------------------------------------------*/
cpl_polynomial **
muse_trace_table_get_polys_for_slice(const cpl_table *aTable,
                                     const unsigned short aSlice)
{
  cpl_ensure(aTable, CPL_ERROR_NULL_INPUT, NULL);
  cpl_ensure(aSlice >= 1 && aSlice <= kMuseSlicesPerCCD,
             CPL_ERROR_ILLEGAL_INPUT, NULL);
  /* search for row containing the requested slice, first to access it *
   * in a possibly incomplete table, and second to check its presence! */
  int irow, nrow = cpl_table_get_nrow(aTable);
  for (irow = 0; irow < nrow; irow++) {
    int err;
    unsigned short slice = cpl_table_get_int(aTable,
                                             MUSE_TRACE_TABLE_COL_SLICE_NO,
                                             irow, &err);
    if (slice == aSlice && !err) {
      break;
    }
  } /* for irow */
  cpl_ensure(irow < nrow, CPL_ERROR_DATA_NOT_FOUND, NULL);

  cpl_polynomial **ptrace = cpl_calloc(MUSE_TRACE_NPOLY,
                                       sizeof(cpl_polynomial *));
  char colname[7]; /* "tcI_NN" plus null byte */
  int ipoly;
  for (ipoly = 0; ipoly < MUSE_TRACE_NPOLY; ipoly++) {
    int traceorder = muse_trace_table_get_order(aTable);
    ptrace[ipoly] = cpl_polynomial_new(1);

    /* fill in the orders of the polynomial */
    int k;
    for (k = 0; k <= traceorder; k++) {
      cpl_size pows[1] = { k }; /* trick to access the polynomial */
      sprintf(colname, MUSE_TRACE_TABLE_COL_COEFF, ipoly, k);
      int err;
      cpl_polynomial_set_coeff(ptrace[ipoly], pows,
                               cpl_table_get(aTable, colname, irow, &err));
      if (err != 0) { /* broken table entry */
        cpl_polynomial_delete(ptrace[MUSE_TRACE_CENTER]);
        cpl_polynomial_delete(ptrace[MUSE_TRACE_LEFT]);
        cpl_polynomial_delete(ptrace[MUSE_TRACE_RIGHT]);
        cpl_free(ptrace);
        cpl_error_set_message(__func__, CPL_ERROR_ILLEGAL_OUTPUT, "Trace table "
                              "broken in slice %hu (row index %d) column %s",
                              aSlice, irow, colname);
        return NULL;
      } /* if */
    } /* for k */
  } /* for ipoly */

  return ptrace;
} /* muse_trace_table_get_polys_for_slice() */

/*---------------------------------------------------------------------------*/
/**
  @brief   Delete the multi-polynomial array created in relation to tracing.
  @param   aPolys   the array of polynomials
 */
/*---------------------------------------------------------------------------*/
void
muse_trace_polys_delete(cpl_polynomial *aPolys[])
{
  if (!aPolys) {
    return;
  }
  cpl_polynomial_delete(aPolys[MUSE_TRACE_CENTER]);
  cpl_polynomial_delete(aPolys[MUSE_TRACE_LEFT]);
  cpl_polynomial_delete(aPolys[MUSE_TRACE_RIGHT]);
  cpl_free(aPolys);
} /* muse_trace_polys_delete() */

/* plot the result of muse_trace_locate_slices() */
static void
muse_trace_plot_located_slices(cpl_vector *aRowVec, cpl_vector *aCenters,
                               double aMedian, double aMDev, double aLimit,
                               const unsigned char aIFU)
{
#if HAVE_POPEN && HAVE_PCLOSE
  FILE *gp = popen("gnuplot -persist", "w");
  if (!gp) {
    cpl_msg_error(__func__, "could not open gnuplot for plotting");
    return;
  }

#if HAVE_MKDTEMP
  char dirtemplate[] = "/tmp/muse_trace_plot_located_slices_XXXXXX";
  char *dirname = mkdtemp(dirtemplate);
  if (!dirname) {
    return;
  }
#else
  char dirname[] = "/tmp";
#endif
  char *out1 = cpl_sprintf("%s/row.dat", dirname);
  FILE *fp = fopen(out1, "w");
  cpl_vector_dump(aRowVec, fp);
  fclose(fp);
  char *out2 = cpl_sprintf("%s/centers.dat", dirname);
  fp = fopen(out2, "w");
  cpl_vector_dump(aCenters, fp);
  fclose(fp);

  fprintf(gp, "set title \"located slices (IFU %hhu): median %.2f+/-%.2f, limit"
          " %.2f\"\nunset key\nset style fill solid 0.5\n", aIFU, aMedian, aMDev,
          aLimit);
  fprintf(gp, "median(x)=%e\nlimit(x)=%e\nlo(x)=%e\n",
          aMedian, aLimit, aMedian - aMDev);
  fprintf(gp, "set xrange [%d:%"CPL_SIZE_FORMAT"]\n", 1, cpl_vector_get_size(aRowVec));
  fprintf(gp, "set yrange [%e:%e]\n", aLimit - 0.5*aMDev, aMedian + 1.3*aMDev);
  fprintf(gp, "plot lo(x) w filledcu y1=%e, "
          "         median(x) t \"median\", limit(x) t \"limit\" w l lw 2, "
          "         \"%s\" w l lt 7, \"%s\" u 2:(%e):1 w p lt -1, "
          "         \"%s\" u 2:(%e):1 w labels\n",
          aMedian+aMDev, out1, out2, aMedian, out2, aMedian+200);

  pclose(gp);
  remove(out1);
  remove(out2);
  cpl_free(out1);
  cpl_free(out2);
#if HAVE_MKDTEMP && HAVE_UNISTD_H
  int rc = rmdir(dirname);
  if (rc < 0) {
    cpl_msg_warning(__func__, "Used %s for plotting, please clean it manually!",
                    dirname);
  }
#endif
#endif /* HAVE_POPEN && HAVE_PCLOSE */
} /* muse_trace_plot_located_slices() */

/*---------------------------------------------------------------------------*/
/**
  @brief   Plotting of trace sample points and solution using gnuplot.
  @param   aSamples   the table with all trace sample points
  @param   aTrace     the table with the tracing solution
  @param   aSlice1    first slice to plot (1 or lower starts with the first one)
  @param   aSlice2    last slice to plot (kMuseSlicesPerCCD or higher ends with
                      the last one)
  @param   aIFU       the IFU number (only used if > 0)
  @param   aImage     the image to use as background for the plot
  @return  CPL_ERROR_NONE on success a CPL error code on failure

  @error{return CPL_ERROR_NULL_INPUT, input aSamples table is NULL}
  @error{return CPL_ERROR_ILLEGAL_INPUT,
         input table is not a MUSE trace samples table}
  @error{return CPL_ERROR_ASSIGNING_STREAM,
         gnuplot stream (pipe) could not be opened}
  @error{return CPL_ERROR_UNSUPPORTED_MODE,
         platform does not have popen() or pclose()}
  @error{set CPL error message to contain failing filename\, return CPL_ERROR_FILE_NOT_CREATED,
         could not open temporary file for plotting of data}
 */
/*---------------------------------------------------------------------------*/
cpl_error_code
muse_trace_plot_samples(cpl_table *aSamples, cpl_table *aTrace,
                        unsigned short aSlice1, unsigned short aSlice2,
                        unsigned char aIFU, muse_image *aImage)
{
#if HAVE_POPEN && HAVE_PCLOSE
  cpl_ensure_code(aSamples, CPL_ERROR_NULL_INPUT);
  cpl_error_code rc = muse_cpltable_check(aSamples, muse_tracesamples_def);
  cpl_ensure_code(rc == CPL_ERROR_NONE, rc);
  /* if errors occur, only plot the two central slices */
  if (aSlice1 < 1 || aSlice1 > kMuseSlicesPerCCD || aSlice1 > aSlice2 ||
      aSlice2 < 1 || aSlice2 > kMuseSlicesPerCCD) {
    fprintf(stderr, "Warning: resetting slice numbers (%hu to %hu does not make"
            " sense)!\n", aSlice1, aSlice2);
    aSlice1 = kMuseSlicesPerCCD / 2;
    aSlice2 = kMuseSlicesPerCCD / 2 + 1;
  }
  if (aSlice2 - aSlice1 > 10) {
    fprintf(stderr, "Warning: plotting %d slices may take a long time and "
            "RAM/disk space!\n", (int)aSlice2 - (int)aSlice1 + 1);
  }
  printf("Plotting ");;
  if (aIFU > 0) {
    printf("IFU %hhu, ", aIFU);
  }
  printf("slices %hu to %hu\n", aSlice1, aSlice2);

  FILE *gp = popen("gnuplot", "w");
  if (!gp) {
    return CPL_ERROR_ASSIGNING_STREAM;
  }

  int nx = -1, ny = kMuseOutputYTop;
  const float *data = NULL;
  if (aImage) {
    nx = cpl_image_get_size_x(aImage->data);
    ny = cpl_image_get_size_x(aImage->data);
    data = cpl_image_get_data_float_const(aImage->data);
  }

  /* fixed output file names for both purposes (plotting of samples and *
   * background) within a hopefully random and protected, directory     */
#if HAVE_MKDTEMP
  char dirtemplate[] = "/tmp/muse_trace_plot_samples_XXXXXX";
  char *dirname = mkdtemp(dirtemplate);
  if (!dirname) {
    return CPL_ERROR_FILE_NOT_CREATED;
  }
#else
  char dirname[] = "/tmp";
#endif
  FILE *tf = NULL, *sf = NULL;
  char *t_out = NULL;
  if (aImage) {
    t_out = cpl_sprintf("%s/muse_trace_plot_flatimage.dat", dirname);
    tf = fopen(t_out, "w+");
    if (!tf) {
      cpl_error_set_message(__func__, CPL_ERROR_FILE_NOT_CREATED, "\"%s\"",
                            t_out);
      cpl_free(t_out); /* fails gracefully on NULL */
      return CPL_ERROR_FILE_NOT_CREATED;
    }
  }
  char *s_out = cpl_sprintf("%s/muse_trace_plot_samples.dat", dirname);
  sf = fopen(s_out, "w+");
  if (!sf) {
    cpl_error_set_message(__func__, CPL_ERROR_FILE_NOT_CREATED, "%s", s_out);
    cpl_free(t_out); /* fails gracefully on NULL */
    cpl_free(s_out);
    return CPL_ERROR_FILE_NOT_CREATED;
  }

  /* plot all relevant slices at once */
  int i, lplot = INT_MAX, rplot = INT_MIN;
  unsigned short nslice;
  for (nslice = aSlice1; nslice <= aSlice2; nslice++) {
    /* write out all relevant sample points */
    for (i = 0; i < cpl_table_get_nrow(aSamples); i++) {
      if (nslice != cpl_table_get_int(aSamples, "slice", i, NULL)) {
        /* not this slice */
        continue;
      }
      float l = cpl_table_get_float(aSamples, "left", i, NULL),
            r = cpl_table_get_float(aSamples, "right", i, NULL);
      fprintf(sf, "%g %g %g %g\n",
              cpl_table_get_float(aSamples, "y", i, NULL),
              cpl_table_get_float(aSamples, "mid", i, NULL), l, r);
      if ((int)floor(l) < lplot) {
        lplot = floor(l);
      }
      if ((int)floor(r) > rplot) {
        rplot = floor(r);
      }
    }
    /* expand the area a bit, to make the edges fully appear on the plot */
    lplot -= 5;
    rplot += 5;

    if (aTrace) {
      int order = muse_trace_table_get_order(aTrace);
      double *c = (double *)cpl_calloc(order + 1, sizeof(double));

      int ipoly;
      for (ipoly = 0; ipoly < MUSE_TRACE_NPOLY; ipoly++) {
        for (i = 0; i <= order; i++) {
          char *colname = cpl_sprintf(MUSE_TRACE_TABLE_COL_COEFF, ipoly, i);
          c[i] = cpl_table_get_double(aTrace, colname, nslice-1, NULL);
          cpl_free(colname);
        }

        /* set up the function for the plot */
        fprintf(gp, "p%02hu%1d(x) = (%g)", nslice, ipoly, c[0]);
        for (i = 1; i <= order; i++) {
          fprintf(gp, " + (%g) * x**(%d)", c[i], i);
        }
        fprintf(gp, "\n");
      } /* for ipoly */
      cpl_free(c);
    } /* if aTrace */
  } /* for nslice */
  if (aImage) {
    /* plot the image data, too */
    for (i = lplot - 1; i < rplot; i++) {
      int j;
      for (j = 0; j < ny; j++) {
        if (i < 0 || i >= nx || j < 0 || j >= ny) {
          continue;
        }
        fprintf(tf, "%d %d %f\n", i+1, j+1, data[i + j*nx]);
      } /* for j (vertical pixels) */
    } /* for i (horizontal pixels) */
    printf("Written \"%s\".\n", t_out);
    fclose(tf);
  }
  printf("Written \"%s\".\n", s_out);
  fclose(sf); /* close the file now to unlock it for plotting */

  /* plot title */
  fprintf(gp, "set title \"trace result, ");
  if (aIFU > 0) {
    fprintf(gp, "IFU %hhu, ", aIFU);
  }
  fprintf(gp, "slices %hu to %hu\"\n", aSlice1, aSlice2);
  fprintf(gp, "set palette gray\n"); /* alternative palette in greyscale */
  fprintf(gp, "unset key\n");

  /* enough sampling points for the vertical dimension */
  fprintf(gp, "set samples %d\n", ny);
  /* we need parametric mode because we want to plot the slices vertically */
  fprintf(gp, "set parametric\n");
  /* set ranges for plotting */
  fprintf(gp, "set xrange [%d:%d]\n", lplot, rplot);
  fprintf(gp, "set yrange [%d:%d]\n", 1, ny);
  fprintf(gp, "set trange [%d:%d]\n", 1, ny);
  /* we are dealing with normalized master flats here */
  fprintf(gp, "set cbrange [1e-4:1.5]\n");

  /* finally create the plot itself */
  fprintf(gp, "plot ");
  if (aImage) {
    fprintf(gp, "\"%s\" w image, ", t_out);
  }
  fprintf(gp, "\"%s\" u 2:1 t \"center points\" w p pt 2 lt rgb \"blue\" ps 1.2, "
              "\"%s\" u 3:1 t \"edge points left\" w p pt 2 lt rgb \"red\" ps 0.8, "
              "\"%s\" u 4:1 t \"edge points right\" w p pt 2 lt rgb \"green\" ps 0.8",
              s_out, s_out, s_out);
  if (aTrace) {
    /* show trace polynomials pnni(x) in parametric mode */
    for (nslice = aSlice1; nslice <= aSlice2; nslice++) {
      fprintf(gp, ", p%02hu0(t),t t \"center\" w l lt rgb \"dark-blue\" lw 2, "
                  "p%02hu1(t),t t \"left\" w l lt rgb \"dark-red\" lw 1, "
                  "p%02hu2(t),t t \"right\" w l lt rgb \"forest-green\" lw 1",
                  nslice, nslice, nslice);
    } /* for nslice */
  }
  fprintf(gp, "\n");
  fflush(gp);
  /* request keypress, so that working with the mouse keeps   *
   * working and gnuplot has enough time to actually draw all *
   * the stuff before the files get removed                   */
  printf("Press ENTER to end program and close plot\n");
  getchar();
  remove(s_out);
  if (aImage) {
    remove(t_out);
    cpl_free(t_out);
  }
  rmdir(dirname);
  cpl_free(s_out);
  pclose(gp);
  return CPL_ERROR_NONE;
#else /* no HAVE_POPEN && HAVE_PCLOSE */
  return CPL_ERROR_UNSUPPORTED_MODE;
#endif /* HAVE_POPEN && HAVE_PCLOSE */
} /* muse_trace_plot_samples() */

/*---------------------------------------------------------------------------*/
/**
  @brief   Plotting the width from trace sample points using gnuplot.
  @param   aSamples   the table with all trace sample points
  @param   aSlice1    first slice to plot (1 or lower starts with the first one)
  @param   aSlice2    last slice to plot (kMuseSlicesPerCCD or higher ends with
                      the last one)
  @param   aIFU       the IFU number (only used if > 0)
  @return  CPL_ERROR_NONE on success a CPL error code on failure

  Uses color-coded lines/points for each relevant slice, ranging from red on
  the leftmost slice to green on the rightmost one.

  @error{return CPL_ERROR_NULL_INPUT, input aSamples table is NULL}
  @error{return CPL_ERROR_ILLEGAL_INPUT,
         input table is not a MUSE trace samples table}
  @error{return CPL_ERROR_ASSIGNING_STREAM,
         gnuplot stream (pipe) could not be opened}
  @error{return CPL_ERROR_UNSUPPORTED_MODE,
         platform does not have popen() or pclose()}
  @error{set CPL error message to contain failing filename\, return CPL_ERROR_FILE_NOT_CREATED,
         could not open temporary file for plotting of data}
 */
/*---------------------------------------------------------------------------*/
cpl_error_code
muse_trace_plot_widths(cpl_table *aSamples, unsigned short aSlice1,
                       unsigned short aSlice2, unsigned char aIFU)
{
#if HAVE_POPEN && HAVE_PCLOSE
  cpl_ensure_code(aSamples, CPL_ERROR_NULL_INPUT);
  cpl_error_code rc = muse_cpltable_check(aSamples, muse_tracesamples_def);
  cpl_ensure_code(rc == CPL_ERROR_NONE, rc);
  /* if errors occur, only plot the two central slices */
  if (aSlice1 < 1 || aSlice1 > kMuseSlicesPerCCD || aSlice1 > aSlice2 ||
      aSlice2 < 1 || aSlice2 > kMuseSlicesPerCCD) {
    fprintf(stderr, "Warning: resetting slice numbers (%hu to %hu does not make"
            " sense)!\n", aSlice1, aSlice2);
    aSlice1 = kMuseSlicesPerCCD / 2;
    aSlice2 = kMuseSlicesPerCCD / 2 + 1;
  }
  printf("Plotting ");;
  if (aIFU > 0) {
    printf("IFU %hhu, ", aIFU);
  }
  printf("slices %hu to %hu\n", aSlice1, aSlice2);

  FILE *gp = popen("gnuplot", "w");
  if (!gp) {
    return CPL_ERROR_ASSIGNING_STREAM;
  }

  int nrow = cpl_table_get_nrow(aSamples);
  const int *sdata = cpl_table_get_data_int_const(aSamples, "slice");
  const float *ydata = cpl_table_get_data_float_const(aSamples, "y"),
              *ldata = cpl_table_get_data_float_const(aSamples, "left"),
              *rdata = cpl_table_get_data_float_const(aSamples, "right");

  /* plot title */
  fprintf(gp, "set title \"trace slice widths, ");
  if (aIFU > 0) {
    fprintf(gp, "IFU %hhu, ", aIFU);
  }
  fprintf(gp, "slices %hu to %hu\"\n", aSlice1, aSlice2);
  fprintf(gp, "set key outside below\n");
  /* set ranges and axes for plotting */
  fprintf(gp, "set xrange [%d:%d]\n", 1, kMuseOutputYTop);
  fprintf(gp, "set yrange [%f:%f]\n", kMuseSliceLoLikelyWidth,
          kMuseSliceHiLikelyWidth);
  fprintf(gp, "set xlabel \"y position on CCD [pix]\"\n");
  fprintf(gp, "set ylabel \"slice width at y position [pix]\"\n");

  /* distribute slices over 256 color values */
  double dslice = (aSlice2 - aSlice1) / 255.;
  if (dslice == 0.) { /* take care not to produce NANs below when dividing */
    dslice = 1.;
  }
  /* finally create the plot itself, loop over all slices */
  fprintf(gp, "plot ");
  unsigned short nslice;
  for (nslice = aSlice1; nslice <= aSlice2; nslice++) {
    /* change color going from left (red) to right (green), *
     * depending on slice number                            */
    fprintf(gp, "\"-\" t \"slice %02hu\" w lp ps 0.8 lt rgb \"#%02x%02x%02x\"",
            nslice,
            (int)((nslice - aSlice1) / dslice), /* red */
            (int)((aSlice2 - nslice) / dslice), /* green */
            0);                                 /* blue */
    if (nslice == aSlice2) {
      fprintf(gp, "\n");
    } else {
      fprintf(gp, ", ");
    }
  } /* for nslice */
  fflush(gp);
  for (nslice = aSlice1; nslice <= aSlice2; nslice++) {
    int i;
    for (i = 0; i < nrow; i++) {
      if (nslice == sdata[i]) {
        fprintf(gp, "%f %f\n", ydata[i], rdata[i]-ldata[i]);
      }
    }
    fprintf(gp, "EOF\n");
  } /* for nslice */
  fprintf(gp, "\n");
  fflush(gp);
  /* request keypress, so that working with the mouse keeps   *
   * working and gnuplot has enough time to actually draw all *
   * the stuff before the files get removed                   */
  printf("Press ENTER to end program and close plot\n");
  getchar();
  pclose(gp);
  return CPL_ERROR_NONE;
#else /* no HAVE_POPEN && HAVE_PCLOSE */
  return CPL_ERROR_UNSUPPORTED_MODE;
#endif /* HAVE_POPEN && HAVE_PCLOSE */
} /* muse_trace_plot_widths() */

/**@}*/
