/* * Copyright (c) 2022 EdgeImpulse Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an "AS * IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either * express or implied. See the License for the specific language * governing permissions and limitations under the License. * * SPDX-License-Identifier: Apache-2.0 */ #ifndef __EIDSP_IMAGE_PROCESSING__H__ #define __EIDSP_IMAGE_PROCESSING__H__ #include "edge-impulse-sdk/dsp/ei_utils.h" #include "edge-impulse-sdk/porting/ei_classifier_porting.h" #include "edge-impulse-sdk/dsp/returntypes.hpp" #include "edge-impulse-sdk/dsp/image/processing.hpp" namespace ei { namespace image { namespace processing { enum YUV_OPTIONS { BIG_ENDIAN_ORDER = 1, //RGB reading from low to high memory. Otherwise, uses native encoding PAD_4B = 2, // pad 0x00 on the high B. ie 0x00RRGGBB }; /** * @brief Convert YUV to RGB * * @param rgb_out Output buffer (can be the same as yuv_in if big enough) * @param yuv_in Input buffer * @param in_size_B Size of input image in B * @param opts Note, only BIG_ENDIAN_ORDER supported presently */ int yuv422_to_rgb888( unsigned char *rgb_out, unsigned const char *yuv_in, unsigned int in_size_B, YUV_OPTIONS opts) { // Clamp out of range values #define EI_CLAMP(t) (((t) > 255) ? 255 : (((t) < 0) ? 0 : (t))) // Color space conversion for RGB #define EI_GET_R_FROM_YUV(y, u, v) ((298 * y + 409 * v + 128) >> 8) #define EI_GET_G_FROM_YUV(y, u, v) ((298 * y - 100 * u - 208 * v + 128) >> 8) #define EI_GET_B_FROM_YUV(y, u, v) ((298 * y + 516 * u + 128) >> 8) unsigned int in_size_pixels = in_size_B / 4; yuv_in += in_size_B - 1; int rgb_end = TEST_BIT_MASK(opts, PAD_4B) ? 2 * in_size_B : (6 * in_size_B) / 4; rgb_out += rgb_end - 1; // Going backwards probably looks strange, but // This allows us to do the algorithm in place! // User needs to put the YUV image into a larger buffer than necessary // But going backwards means we don't overwrite the YUV bytes // until we don't need them anymore for (unsigned int i = 0; i < in_size_pixels; ++i) { int y2 = *yuv_in-- - 16; int v = *yuv_in-- - 128; int y0 = *yuv_in-- - 16; int u0 = *yuv_in-- - 128; if (TEST_BIT_MASK(opts, BIG_ENDIAN_ORDER)) { *rgb_out-- = EI_CLAMP(EI_GET_B_FROM_YUV(y2, u0, v)); *rgb_out-- = EI_CLAMP(EI_GET_G_FROM_YUV(y2, u0, v)); *rgb_out-- = EI_CLAMP(EI_GET_R_FROM_YUV(y2, u0, v)); if (TEST_BIT_MASK(opts, PAD_4B)) { *rgb_out-- = 0; } *rgb_out-- = EI_CLAMP(EI_GET_B_FROM_YUV(y0, u0, v)); *rgb_out-- = EI_CLAMP(EI_GET_G_FROM_YUV(y0, u0, v)); *rgb_out-- = EI_CLAMP(EI_GET_R_FROM_YUV(y0, u0, v)); if (TEST_BIT_MASK(opts, PAD_4B)) { *rgb_out-- = 0; } } else { // not yet supported return EIDSP_NOT_SUPPORTED; } } return EIDSP_OK; } /** * @brief Crops an image. Can be in-place. 4B alignment for best performance * (Alignment is tested, will fall back to B by B movement) * * @param srcWidth X dimension in pixels * @param srcHeight Y dimension in pixels * @param srcImage Input buffer * @param startX X coord of first pixel to keep * @param startY Y coord of the first pixel to keep * @param dstWidth Desired X dimension in pixels (should be smaller than srcWidth) * @param dstHeight Desired Y dimension in pixels (should be smaller than srcHeight) * @param dstImage Output buffer, can be the same as srcImage * @param iBpp 8 or 16 for bits per pixel */ int cropImage( const uint8_t *srcImage, int srcWidth, int srcHeight, int startX, int startY, uint8_t *dstImage, int dstWidth, int dstHeight, int iBpp) { uint32_t *s32, *d32; int x, y; if (startX < 0 || startX >= srcWidth || startY < 0 || startY >= srcHeight || (startX + dstWidth) > srcWidth || (startY + dstHeight) > srcHeight) { return EIDSP_PARAMETER_INVALID; // invalid parameters } if (iBpp != 8 && iBpp != 16) { return EIDSP_PARAMETER_INVALID; } if (iBpp == 8) { const uint8_t *s; uint8_t *d; for (y = 0; y < dstHeight; y++) { s = &srcImage[srcWidth * (y + startY) + startX]; d = &dstImage[(dstWidth * y)]; x = 0; if ((intptr_t)s & 3 || (intptr_t)d & 3) { // either src or dst pointer is not aligned for (; x < dstWidth; x++) { *d++ = *s++; // have to do it byte-by-byte } } else { // move 4 bytes at a time if aligned or alignment not enforced s32 = (uint32_t *)s; d32 = (uint32_t *)d; for (; x < dstWidth - 3; x += 4) { *d32++ = *s32++; } // any remaining stragglers? s = (uint8_t *)s32; d = (uint8_t *)d32; for (; x < dstWidth; x++) { *d++ = *s++; } } } // for y } // 8-bpp else { uint16_t *s, *d; for (y = 0; y < dstHeight; y++) { s = (uint16_t *)&srcImage[2 * srcWidth * (y + startY) + startX * 2]; d = (uint16_t *)&dstImage[(dstWidth * y * 2)]; x = 0; if ((intptr_t)s & 2 || (intptr_t)d & 2) { // either src or dst pointer is not aligned for (; x < dstWidth; x++) { *d++ = *s++; // have to do it 16-bits at a time } } else { // move 4 bytes at a time if aligned or alignment no enforced s32 = (uint32_t *)s; d32 = (uint32_t *)d; for (; x < dstWidth - 1; x += 2) { // we can move 2 pixels at a time *d32++ = *s32++; } // any remaining stragglers? s = (uint16_t *)s32; d = (uint16_t *)d32; for (; x < dstWidth; x++) { *d++ = *s++; } } } // for y } // 16-bpp case return EIDSP_OK; } /* cropImage() */ /** * @copydoc cropImage( int srcWidth, int srcHeight, const uint8_t *srcImage, int startX, int startY, int dstWidth, int dstHeight, uint8_t *dstImage, int iBpp) */ int crop_image_rgb888_packed( const uint8_t *srcImage, int srcWidth, int srcHeight, int startX, int startY, uint8_t *dstImage, int dstWidth, int dstHeight) { // use 8 bpp mode, but do everything *3 for RGB return cropImage( srcImage, srcWidth * 3, srcHeight, startX * 3, startY, dstImage, dstWidth * 3, dstHeight, 8); } /** * @brief Resize an image using interpolation * Can be used to resize the image smaller or larger * If resizing much smaller than 1/3 size, then a more rubust algorithm should average all of the pixels * This algorithm uses bilinear interpolation - averages a 2x2 region to generate each new pixel * * @param srcWidth Input image width in pixels * @param srcHeight Input image height in pixels * @param srcImage Input buffer * @param dstWidth Output image width in pixels * @param dstHeight Output image height in pixels * @param dstImage Output buffer, can be same as input buffer * @param pixel_size_B Size of pixels in Bytes. 3 for RGB, 1 for mono */ int resize_image( const uint8_t *srcImage, int srcWidth, int srcHeight, uint8_t *dstImage, int dstWidth, int dstHeight, int pixel_size_B) { // Copied from ei_camera.cpp in firmware-eta-compute // Modified for RGB888 // This needs to be < 16 or it won't fit. Cortex-M4 only has SIMD for signed multiplies constexpr int FRAC_BITS = 14; constexpr int FRAC_VAL = (1 << FRAC_BITS); constexpr int FRAC_MASK = (FRAC_VAL - 1); uint32_t src_x_accum, src_y_accum; // accumulators and fractions for scaling the image uint32_t x_frac, nx_frac, y_frac, ny_frac; int x, y, ty; if (srcHeight < 2) { return EIDSP_PARAMETER_INVALID; } // start at 1/2 pixel in to account for integer downsampling which might miss pixels src_y_accum = FRAC_VAL / 2; const uint32_t src_x_frac = (srcWidth * FRAC_VAL) / dstWidth; const uint32_t src_y_frac = (srcHeight * FRAC_VAL) / dstHeight; //from here out, *3 b/c RGB srcWidth *= pixel_size_B; //srcHeight not used for indexing //dstWidth still needed as is //dstHeight shouldn't be scaled const uint8_t *s; uint8_t *d; for (y = 0; y < dstHeight; y++) { // do indexing computations ty = src_y_accum >> FRAC_BITS; // src y y_frac = src_y_accum & FRAC_MASK; src_y_accum += src_y_frac; ny_frac = FRAC_VAL - y_frac; // y fraction and 1.0 - y fraction s = &srcImage[ty * srcWidth]; d = &dstImage[y * dstWidth * pixel_size_B]; //not scaled above // start at 1/2 pixel in to account for integer downsampling which might miss pixels src_x_accum = FRAC_VAL / 2; for (x = 0; x < dstWidth; x++) { uint32_t tx, p00, p01, p10, p11; // do indexing computations tx = (src_x_accum >> FRAC_BITS) * pixel_size_B; x_frac = src_x_accum & FRAC_MASK; nx_frac = FRAC_VAL - x_frac; // x fraction and 1.0 - x fraction src_x_accum += src_x_frac; //interpolate and write out for (int color = 0; color < pixel_size_B; color++) // do pixel_size_B times for pixel_size_B colors { p00 = s[tx]; p10 = s[tx + pixel_size_B]; p01 = s[tx + srcWidth]; p11 = s[tx + srcWidth + pixel_size_B]; p00 = ((p00 * nx_frac) + (p10 * x_frac) + FRAC_VAL / 2) >> FRAC_BITS; // top line p01 = ((p01 * nx_frac) + (p11 * x_frac) + FRAC_VAL / 2) >> FRAC_BITS; // bottom line p00 = ((p00 * ny_frac) + (p01 * y_frac) + FRAC_VAL / 2) >> FRAC_BITS; //top + bottom *d++ = (uint8_t)p00; // store new pixel //ready next loop tx++; } } // for x } // for y return EIDSP_OK; } // resizeImage() /** * @brief Calculate new dims that match the aspect ratio of destination * This prevents a squashed look * The smallest axis is held constant * * @param srcWidth Input width in pixels * @param srcHeight Input height in pixels * @param dstWidth Ultimate width in pixels * @param dstHeight Ultimate height in pixels * @param[out] cropWidth Width in pixels that matches the aspect ratio * @param[out] cropHeight Height in pixels that matches the aspect ratio */ void calculate_crop_dims( int srcWidth, int srcHeight, int dstWidth, int dstHeight, int &cropWidth, int &cropHeight) { //first, trim the largest axis to match destination aspect ratio //calculate by fixing the smaller axis if (srcWidth > srcHeight) { cropWidth = (uint32_t)(dstWidth * srcHeight) / dstHeight; //cast in case int is small cropHeight = srcHeight; } else { cropHeight = (uint32_t)(dstHeight * srcWidth) / dstWidth; cropWidth = srcWidth; } } int crop_and_interpolate_rgb888( const uint8_t *srcImage, int srcWidth, int srcHeight, uint8_t *dstImage, int dstWidth, int dstHeight) { int cropWidth, cropHeight; // What are dimensions that maintain aspect ratio? calculate_crop_dims(srcWidth, srcHeight, dstWidth, dstHeight, cropWidth, cropHeight); // Now crop to that dimension int res = crop_image_rgb888_packed( srcImage, srcWidth, srcHeight, (srcWidth - cropWidth) / 2, (srcHeight - cropHeight) / 2, dstImage, cropWidth, cropHeight); if( res != EIDSP_OK) { return res; } // Finally, interpolate down to desired dimensions, in place return resize_image(dstImage, cropWidth, cropHeight, dstImage, dstWidth, dstHeight, 3); } int crop_and_interpolate_image( const uint8_t *srcImage, int srcWidth, int srcHeight, uint8_t *dstImage, int dstWidth, int dstHeight, int pixel_size_B) { int cropWidth, cropHeight; // What are dimensions that maintain aspect ratio? calculate_crop_dims(srcWidth, srcHeight, dstWidth, dstHeight, cropWidth, cropHeight); // Now crop to that dimension int res = cropImage( srcImage, srcWidth * pixel_size_B, srcHeight, ((srcWidth - cropWidth) / 2) * pixel_size_B, (srcHeight - cropHeight) / 2, dstImage, cropWidth * pixel_size_B, cropHeight, 8); if( res != EIDSP_OK) { return res; } // Finally, interpolate down to desired dimensions, in place return resize_image(dstImage, cropWidth, cropHeight, dstImage, dstWidth, dstHeight, pixel_size_B); } }}} //namespaces #endif //!__EI_IMAGE_PROCESSING__H__