Fix null pointer dereference in echo-message

The bug was introduced while fixing #1705. If a client did not enable
echo-message, and doesn't have a network, it crashes.

Thanks to LunarBNC for reporting this

@@ -889,7 +889,7 @@ void CClient::EchoMessage(const CMessage& Message) {
     CMessage EchoedMessage = Message;
     for (CClient* pClient : GetClients()) {
         if (pClient->HasEchoMessage() ||
-            (pClient != this && (m_pNetwork->IsChan(Message.GetParam(0)) ||
+            (pClient != this && ((m_pNetwork && m_pNetwork->IsChan(Message.GetParam(0))) ||
                                  pClient->HasSelfMessage()))) {
             EchoedMessage.SetNick(GetNickMask());
             pClient->PutClient(EchoedMessage);


Properly store certificate exceptions in Java viewer

Like the native viewer, the Java viewer didn't store certificate
exceptions properly. Whilst not as bad as the native viewer, it still
failed to check that a stored certificate wouldn't be maliciously used
for another server. In practice this can in most cases be used to
impersonate another server.

Handle this like the native viewer by storing exceptions for a specific
hostname/certificate combination.

@@ -107,12 +107,6 @@ public static void setDefaults()
       X509CRL.setDefaultStr(getDefaultCRL());
   }

-// FIXME:
-// Need to shutdown the connection cleanly
-
-// FIXME?
-// add a finalizer method that calls shutdown
-
   public boolean processMsg(CConnection cc) {
     is = (FdInStream)cc.getInStream();
     os = (FdOutStream)cc.getOutStream();

@@ -269,8 +263,13 @@ public void checkServerTrusted(X509Certificate[] chain, String authType)
     {
       Collection<? extends Certificate> certs = null;
       X509Certificate cert = chain[0];
+      String pk =
+        Base64.getEncoder().encodeToString(cert.getPublicKey().getEncoded());
       try {
         cert.checkValidity();
+        verifyHostname(cert);
+      } catch(CertificateParsingException e) {
+        throw new SystemException(e.getMessage());
       } catch(CertificateNotYetValidException e) {
         throw new AuthFailureException("server certificate has not been activated");
       } catch(CertificateExpiredException e) {

@@ -399,12 +436,13 @@ private void verifyHostname(X509Certificate cert)
         }
         Object[] answer = {"YES", "NO"};
         int ret = JOptionPane.showOptionDialog(null,
-          "Hostname verification failed. Do you want to continue?",
-          "Hostname Verification Failure",
+          "Hostname ("+client.getServerName()+") does not match the"+
+          " server certificate, do you want to continue?",
+          "Certificate hostname mismatch",
           JOptionPane.YES_NO_OPTION, JOptionPane.WARNING_MESSAGE,
           null, answer, answer[0]);
         if (ret != JOptionPane.YES_OPTION)
-          throw new WarningException("Hostname verification failed.");
+          throw new WarningException("Certificate hostname mismatch.");
       } catch (CertificateParsingException e) {
         throw new SystemException(e.getMessage());
       } catch (InvalidNameException e) {


[tflite]: Insert `nullptr` checks when obtaining tensors.

As part of ongoing refactoring, `tflite::GetInput`, `tflite::GetOutput`, `tflite::GetTemporary` and `tflite::GetIntermediates` will return `nullptr` in some cases. Hence, we insert the `nullptr` checks on all usages.

We also insert `nullptr` checks on usages of `tflite::GetVariableInput` and `tflite::GetOptionalInputTensor` but only in the cases where there is no obvious check that `nullptr` is acceptable (that is, we only insert the check for the output of these two functions if the tensor is accessed as if it is always not `nullptr`).

PiperOrigin-RevId: 332521299
Change-Id: I29af455bcb48d0b92e58132d951a3badbd772d56

@@ -58,15 +58,19 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   TF_LITE_ENSURE_EQ(context, NumInputs(node), 2);
   TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);

-  const TfLiteTensor* input = GetInput(context, node, kInputTensor);
-  const TfLiteTensor* axis = GetInput(context, node, kAxis);
+  const TfLiteTensor* input;
+  TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, kInputTensor, &input));
+  const TfLiteTensor* axis;
+  TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, kAxis, &axis));
   // Make sure the axis is only 1 dimension.
   TF_LITE_ENSURE_EQ(context, NumElements(axis), 1);
   // Make sure the axis is only either int32 or int64.
   TF_LITE_ENSURE(context,
                  axis->type == kTfLiteInt32 || axis->type == kTfLiteInt64);

-  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+  TfLiteTensor* output;
+  TF_LITE_ENSURE_OK(context,
+                    GetOutputSafe(context, node, kOutputTensor, &output));

   auto* params = reinterpret_cast<TfLiteArgMaxParams*>(node->builtin_data);
   switch (params->output_type) {
@@ -119,9 +123,13 @@ std::function<bool(T, T)> GetComparefunction(bool is_arg_max) {
 }

 TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node, bool is_arg_max) {
-  const TfLiteTensor* input = GetInput(context, node, kInputTensor);
-  const TfLiteTensor* axis = GetInput(context, node, kAxis);
-  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+  const TfLiteTensor* input;
+  TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, kInputTensor, &input));
+  const TfLiteTensor* axis;
+  TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, kAxis, &axis));
+  TfLiteTensor* output;
+  TF_LITE_ENSURE_OK(context,
+                    GetOutputSafe(context, node, kOutputTensor, &output));
   if (IsDynamicTensor(output)) {
     TF_LITE_ENSURE_STATUS(ResizeOutput(context, input, axis, output));
   }


@@ -45,7 +45,8 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {

   // The number of dimensions of the input tensors must match, and all
   // dimensions except 'axis' must be equal.
-  const TfLiteTensor* t0 = GetInput(context, node, 0);
+  const TfLiteTensor* t0;
+  TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, 0, &t0));
   TfLiteType input_type = t0->type;
   if (axis < 0) axis += t0->dims->size;
   TF_LITE_ENSURE(context, axis >= 0);
@@ -63,7 +64,8 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   // will be the sum of inputs
   int sum_axis = t0->dims->data[axis];
   for (int i = 1; i < num_inputs; ++i) {
-    const TfLiteTensor* t = GetInput(context, node, i);
+    const TfLiteTensor* t;
+    TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, i, &t));
     TF_LITE_ENSURE_EQ(context, t->dims->size, t0->dims->size);
     TF_LITE_ENSURE_EQ(context, t->type, input_type);
     for (int d = 0; d < t0->dims->size; ++d) {
@@ -80,15 +82,17 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
     output_size->data[d] = (d == axis) ? sum_axis : t0->dims->data[d];
   }

-  TfLiteTensor* output = GetOutput(context, node, 0);
+  TfLiteTensor* output;
+  TF_LITE_ENSURE_OK(context, GetOutputSafe(context, node, 0, &output));
   TF_LITE_ENSURE_TYPES_EQ(context, output->type, input_type);

   if (input_type == kTfLiteInt8) {
     // Make sure there is no re-scaling needed for Int8 quantized kernel. This
     // is a restriction we introduced to Int8 kernels.
     VectorOfTensors<int8_t> all_inputs(*context, *node->inputs);
     for (int i = 0; i < node->inputs->size; ++i) {
-      const TfLiteTensor* t = GetInput(context, node, i);
+      const TfLiteTensor* t;
+      TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, i, &t));
       TF_LITE_ENSURE_EQ(context, t->params.scale, output->params.scale);
       TF_LITE_ENSURE_EQ(context, t->params.zero_point,
                         output->params.zero_point);
@@ -103,7 +107,8 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   auto* params =
       reinterpret_cast<TfLiteConcatenationParams*>(node->builtin_data);
   int axis = params->axis;
-  TfLiteTensor* output = GetOutput(context, node, 0);
+  TfLiteTensor* output;
+  TF_LITE_ENSURE_OK(context, GetOutputSafe(context, node, 0, &output));
   if (axis < 0) axis += output->dims->size;

 // TODO(ahentz): Creating 'all_inputs' below is not very efficient. We should


@@ -72,8 +72,10 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   TF_LITE_ENSURE_EQ(context, NumInputs(node), 2);
   TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);

-  const TfLiteTensor* dims = GetInput(context, node, kDimsTensor);
-  const TfLiteTensor* value = GetInput(context, node, kValueTensor);
+  const TfLiteTensor* dims;
+  TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, kDimsTensor, &dims));
+  const TfLiteTensor* value;
+  TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, kValueTensor, &value));

   // Make sure the 1st input tensor is 1-D.
   TF_LITE_ENSURE_EQ(context, NumDimensions(dims), 1);
@@ -85,7 +87,9 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   // Make sure the 2nd input tensor is a scalar.
   TF_LITE_ENSURE_EQ(context, NumDimensions(value), 0);

-  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+  TfLiteTensor* output;
+  TF_LITE_ENSURE_OK(context,
+                    GetOutputSafe(context, node, kOutputTensor, &output));
   output->type = value->type;

   if (IsConstantTensor(dims)) {
@@ -111,12 +115,16 @@ TfLiteStatus FillString(const TfLiteTensor* value, TfLiteTensor* output) {
 }

 TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
-  const TfLiteTensor* value = GetInput(context, node, kValueTensor);
+  const TfLiteTensor* value;
+  TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, kValueTensor, &value));

-  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+  TfLiteTensor* output;
+  TF_LITE_ENSURE_OK(context,
+                    GetOutputSafe(context, node, kOutputTensor, &output));

   if (IsDynamicTensor(output)) {
-    const TfLiteTensor* dims = GetInput(context, node, kDimsTensor);
+    const TfLiteTensor* dims;
+    TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, kDimsTensor, &dims));
     TF_LITE_ENSURE_OK(context, ResizeOutput(context, dims, output));
   }
 #define TF_LITE_FILL(data_type)                                               \


@@ -39,8 +39,11 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   TF_LITE_ENSURE_EQ(context, NumInputs(node), 1);
   TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);

-  const TfLiteTensor* input = GetInput(context, node, kInputTensor);
-  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+  const TfLiteTensor* input;
+  TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, kInputTensor, &input));
+  TfLiteTensor* output;
+  TF_LITE_ENSURE_OK(context,
+                    GetOutputSafe(context, node, kOutputTensor, &output));

   TF_LITE_ENSURE_EQ(context, NumDimensions(input), 4);

@@ -61,8 +64,11 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   auto* params =
       reinterpret_cast<TfLiteLocalResponseNormParams*>(node->builtin_data);

-  const TfLiteTensor* input = GetInput(context, node, kInputTensor);
-  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+  const TfLiteTensor* input;
+  TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, kInputTensor, &input));
+  TfLiteTensor* output;
+  TF_LITE_ENSURE_OK(context,
+                    GetOutputSafe(context, node, kOutputTensor, &output));

   if (output->type == kTfLiteFloat32) {
 #define TF_LITE_LOCAL_RESPONSE_NORM(type)                            \


@@ -79,20 +79,25 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   }

   // Boxes & Scores.
-  const TfLiteTensor* input_boxes = GetInput(context, node, kInputTensorBoxes);
+  const TfLiteTensor* input_boxes;
+  TF_LITE_ENSURE_OK(
+      context, GetInputSafe(context, node, kInputTensorBoxes, &input_boxes));
   TF_LITE_ENSURE_EQ(context, input_boxes->type, kTfLiteFloat32);
   TF_LITE_ENSURE_EQ(context, NumDimensions(input_boxes), 2);
   TF_LITE_ENSURE_EQ(context, SizeOfDimension(input_boxes, 1), 4);
   const int num_boxes = SizeOfDimension(input_boxes, 0);
-  const TfLiteTensor* input_scores =
-      GetInput(context, node, kInputTensorScores);
+  const TfLiteTensor* input_scores;
+  TF_LITE_ENSURE_OK(
+      context, GetInputSafe(context, node, kInputTensorScores, &input_scores));
   TF_LITE_ENSURE_EQ(context, input_scores->type, kTfLiteFloat32);
   TF_LITE_ENSURE_EQ(context, NumDimensions(input_scores), 1);
   TF_LITE_ENSURE_EQ(context, num_boxes, SizeOfDimension(input_scores, 0));

   // Max output size.
-  const TfLiteTensor* input_max_output_size =
-      GetInput(context, node, kInputTensorMaxOutputSize);
+  const TfLiteTensor* input_max_output_size;
+  TF_LITE_ENSURE_OK(context,
+                    GetInputSafe(context, node, kInputTensorMaxOutputSize,
+                                 &input_max_output_size));
   TF_LITE_ENSURE_EQ(context, input_max_output_size->type, kTfLiteInt32);
   TF_LITE_ENSURE_EQ(context, NumDimensions(input_max_output_size), 0);
   const bool is_max_output_size_const = IsConstantTensor(input_max_output_size);
@@ -103,30 +108,43 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   }

   // IoU & Score thresholds.
-  const TfLiteTensor* input_iou_threshold =
-      GetInput(context, node, kInputTensorIouThreshold);
+  const TfLiteTensor* input_iou_threshold;
+  TF_LITE_ENSURE_OK(context,
+                    GetInputSafe(context, node, kInputTensorIouThreshold,
+                                 &input_iou_threshold));
   TF_LITE_ENSURE_EQ(context, input_iou_threshold->type, kTfLiteFloat32);
   TF_LITE_ENSURE_EQ(context, NumDimensions(input_iou_threshold), 0);
-  const TfLiteTensor* input_score_threshold =
-      GetInput(context, node, kInputTensorScoreThreshold);
+  const TfLiteTensor* input_score_threshold;
+  TF_LITE_ENSURE_OK(context,
+                    GetInputSafe(context, node, kInputTensorScoreThreshold,
+                                 &input_score_threshold));
   TF_LITE_ENSURE_EQ(context, input_iou_threshold->type, kTfLiteFloat32);
   TF_LITE_ENSURE_EQ(context, NumDimensions(input_score_threshold), 0);

   if (is_soft_nms) {
-    const TfLiteTensor* input_sigma =
-        GetInput(context, node, kInputTensorSigma);
+    const TfLiteTensor* input_sigma;
+    TF_LITE_ENSURE_OK(
+        context, GetInputSafe(context, node, kInputTensorSigma, &input_sigma));
     TF_LITE_ENSURE_EQ(context, input_sigma->type, kTfLiteFloat32);
     TF_LITE_ENSURE_EQ(context, NumDimensions(input_sigma), 0);

     TF_LITE_ENSURE_EQ(context, NumOutputs(node), 3);
-    TfLiteTensor* output_selected_indices =
-        GetOutput(context, node, kSoftNMSOutputTensorSelectedIndices);
+    TfLiteTensor* output_selected_indices;
+    TF_LITE_ENSURE_OK(
+        context,
+        GetOutputSafe(context, node, kSoftNMSOutputTensorSelectedIndices,
+                      &output_selected_indices));
     output_selected_indices->type = kTfLiteInt32;
-    TfLiteTensor* output_selected_scores =
-        GetOutput(context, node, kSoftNMSOutputTensorSelectedScores);
+    TfLiteTensor* output_selected_scores;
+    TF_LITE_ENSURE_OK(context, GetOutputSafe(context, node,
+                                             kSoftNMSOutputTensorSelectedScores,
+                                             &output_selected_scores));
     output_selected_scores->type = kTfLiteFloat32;
-    TfLiteTensor* output_num_selected_indices =
-        GetOutput(context, node, kSoftNMSOutputTensorNumSelectedIndices);
+    TfLiteTensor* output_num_selected_indices;
+    TF_LITE_ENSURE_OK(
+        context,
+        GetOutputSafe(context, node, kSoftNMSOutputTensorNumSelectedIndices,
+                      &output_num_selected_indices));
     output_num_selected_indices->type = kTfLiteInt32;
     SetTensorSizes(context, output_num_selected_indices, {});

@@ -139,11 +157,15 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
     }
   } else {
     TF_LITE_ENSURE_EQ(context, NumOutputs(node), 2);
-    TfLiteTensor* output_selected_indices =
-        GetOutput(context, node, kNMSOutputTensorSelectedIndices);
+    TfLiteTensor* output_selected_indices;
+    TF_LITE_ENSURE_OK(
+        context, GetOutputSafe(context, node, kNMSOutputTensorSelectedIndices,
+                               &output_selected_indices));
     output_selected_indices->type = kTfLiteInt32;
-    TfLiteTensor* output_num_selected_indices =
-        GetOutput(context, node, kNMSOutputTensorNumSelectedIndices);
+    TfLiteTensor* output_num_selected_indices;
+    TF_LITE_ENSURE_OK(context, GetOutputSafe(context, node,
+                                             kNMSOutputTensorNumSelectedIndices,
+                                             &output_num_selected_indices));
     output_num_selected_indices->type = kTfLiteInt32;
     SetTensorSizes(context, output_num_selected_indices, {});

@@ -179,42 +201,57 @@ void ResetUnusedElementsToZeroes(const int max_output_size,
 TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   const bool is_soft_nms = NumInputs(node) == 6;

-  const TfLiteTensor* input_boxes = GetInput(context, node, kInputTensorBoxes);
+  const TfLiteTensor* input_boxes;
+  TF_LITE_ENSURE_OK(
+      context, GetInputSafe(context, node, kInputTensorBoxes, &input_boxes));
   const int num_boxes = SizeOfDimension(input_boxes, 0);
-  const TfLiteTensor* input_scores =
-      GetInput(context, node, kInputTensorScores);
-  const TfLiteTensor* input_max_output_size =
-      GetInput(context, node, kInputTensorMaxOutputSize);
+  const TfLiteTensor* input_scores;
+  TF_LITE_ENSURE_OK(
+      context, GetInputSafe(context, node, kInputTensorScores, &input_scores));
+  const TfLiteTensor* input_max_output_size;
+  TF_LITE_ENSURE_OK(context,
+                    GetInputSafe(context, node, kInputTensorMaxOutputSize,
+                                 &input_max_output_size));
   const int max_output_size_value = *GetTensorData<int>(input_max_output_size);
   TF_LITE_ENSURE(context, (max_output_size_value >= 0));
   const bool is_max_output_size_const = IsConstantTensor(input_max_output_size);
-  const TfLiteTensor* input_iou_threshold =
-      GetInput(context, node, kInputTensorIouThreshold);
+  const TfLiteTensor* input_iou_threshold;
+  TF_LITE_ENSURE_OK(context,
+                    GetInputSafe(context, node, kInputTensorIouThreshold,
+                                 &input_iou_threshold));
   const float iou_threshold = *GetTensorData<float>(input_iou_threshold);
-  const TfLiteTensor* input_score_threshold =
-      GetInput(context, node, kInputTensorScoreThreshold);
+  const TfLiteTensor* input_score_threshold;
+  TF_LITE_ENSURE_OK(context,
+                    GetInputSafe(context, node, kInputTensorScoreThreshold,
+                                 &input_score_threshold));
   const float score_threshold = *GetTensorData<float>(input_score_threshold);

   TfLiteTensor* output_selected_indices = nullptr;
   TfLiteTensor* output_selected_scores = nullptr;
   TfLiteTensor* output_num_selected_indices = nullptr;

   if (is_soft_nms) {
-    const TfLiteTensor* input_sigma =
-        GetInput(context, node, kInputTensorSigma);
+    const TfLiteTensor* input_sigma;
+    TF_LITE_ENSURE_OK(
+        context, GetInputSafe(context, node, kInputTensorSigma, &input_sigma));
     const float soft_nms_sigma = *GetTensorData<float>(input_sigma);
     if (soft_nms_sigma < 0) {
       context->ReportError(context, "Invalid sigma value for soft NMS: %f",
                            soft_nms_sigma);
       return kTfLiteError;
     }

-    output_selected_indices =
-        GetOutput(context, node, kSoftNMSOutputTensorSelectedIndices);
-    output_selected_scores =
-        GetOutput(context, node, kSoftNMSOutputTensorSelectedScores);
-    output_num_selected_indices =
-        GetOutput(context, node, kSoftNMSOutputTensorNumSelectedIndices);
+    TF_LITE_ENSURE_OK(
+        context,
+        GetOutputSafe(context, node, kSoftNMSOutputTensorSelectedIndices,
+                      &output_selected_indices));
+    TF_LITE_ENSURE_OK(context, GetOutputSafe(context, node,
+                                             kSoftNMSOutputTensorSelectedScores,
+                                             &output_selected_scores));
+    TF_LITE_ENSURE_OK(
+        context,
+        GetOutputSafe(context, node, kSoftNMSOutputTensorNumSelectedIndices,
+                      &output_num_selected_indices));
     if (!is_max_output_size_const) {
       SetTensorSizes(context, output_selected_indices, {max_output_size_value});
       SetTensorSizes(context, output_selected_scores, {max_output_size_value});
@@ -228,10 +265,12 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
         max_output_size_value, *output_num_selected_indices->data.i32,
         output_selected_indices->data.i32, output_selected_scores->data.f);
   } else {
-    output_selected_indices =
-        GetOutput(context, node, kNMSOutputTensorSelectedIndices);
-    output_num_selected_indices =
-        GetOutput(context, node, kNMSOutputTensorNumSelectedIndices);
+    TF_LITE_ENSURE_OK(
+        context, GetOutputSafe(context, node, kNMSOutputTensorSelectedIndices,
+                               &output_selected_indices));
+    TF_LITE_ENSURE_OK(context, GetOutputSafe(context, node,
+                                             kNMSOutputTensorNumSelectedIndices,
+                                             &output_num_selected_indices));
     if (!is_max_output_size_const) {
       SetTensorSizes(context, output_selected_indices, {max_output_size_value});
     }


@@ -38,8 +38,11 @@ TfLiteStatus ResizeOutput(TfLiteContext* context, TfLiteNode* node) {
   std::unique_ptr<TfLiteIntArray, void (*)(TfLiteIntArray*)>
       scoped_output_shape(output_shape, TfLiteIntArrayFree);

-  const TfLiteTensor* input = GetInput(context, node, kInputTensor);
-  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+  const TfLiteTensor* input;
+  TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, kInputTensor, &input));
+  TfLiteTensor* output;
+  TF_LITE_ENSURE_OK(context,
+                    GetOutputSafe(context, node, kOutputTensor, &output));

   // Tensorflow's Reshape allows one of the shape components to have the
   // special -1 value, meaning it will be calculated automatically based on the
@@ -70,6 +73,7 @@ TfLiteStatus ResizeOutput(TfLiteContext* context, TfLiteNode* node) {
 inline TfLiteIntArray* GetOutputShapeFromTensor(TfLiteContext* context,
                                                 TfLiteNode* node) {
   const TfLiteTensor* shape = GetInput(context, node, kShapeTensor);
+  if (shape == nullptr) return nullptr;

   TfLiteIntArray* output_shape = TfLiteIntArrayCreate(shape->dims->data[0]);
   for (int i = 0; i < output_shape->size; ++i) {
@@ -103,7 +107,8 @@ inline TfLiteIntArray* GetOutputShapeFromParam(TfLiteContext* context,
 // Check if the shape tensor is valid. Shapes should be int32 vectors.
 inline bool ShapeIsVector(TfLiteContext* context, TfLiteNode* node) {
   const TfLiteTensor* shape = GetInput(context, node, kShapeTensor);
-  return (shape->dims->size == 1 && shape->type == kTfLiteInt32);
+  return (shape != nullptr && shape->dims->size == 1 &&
+          shape->type == kTfLiteInt32);
 }

 TfLiteIntArray* GetOutputShape(TfLiteContext* context, TfLiteNode* node) {
@@ -122,7 +127,9 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   // calculate their shapes now. String tensors don't benefit from having their
   // shapes precalculated because the actual memory can only be allocated after
   // we know all the content.
-  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+  TfLiteTensor* output;
+  TF_LITE_ENSURE_OK(context,
+                    GetOutputSafe(context, node, kOutputTensor, &output));
   if (output->type != kTfLiteString) {
     if (NumInputs(node) == 1 ||
         IsConstantTensor(GetInput(context, node, kShapeTensor))) {
@@ -135,8 +142,11 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
 }

 TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
-  const TfLiteTensor* input = GetInput(context, node, kInputTensor);
-  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+  const TfLiteTensor* input;
+  TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, kInputTensor, &input));
+  TfLiteTensor* output;
+  TF_LITE_ENSURE_OK(context,
+                    GetOutputSafe(context, node, kOutputTensor, &output));

   // There are two ways in which the 'output' can be made dynamic: it could be
   // a string tensor, or its shape cannot be calculated during Prepare(). In


@@ -40,8 +40,11 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   TF_LITE_ENSURE_EQ(context, NumInputs(node), 1);
   TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);

-  const TfLiteTensor* input = GetInput(context, node, kInputTensor);
-  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+  const TfLiteTensor* input;
+  TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, kInputTensor, &input));
+  TfLiteTensor* output;
+  TF_LITE_ENSURE_OK(context,
+                    GetOutputSafe(context, node, kOutputTensor, &output));

   auto* params = reinterpret_cast<TfLiteShapeParams*>(node->builtin_data);
   switch (params->out_type) {


@@ -49,9 +49,14 @@ TfLiteIntArray* MultiplyShapeDims(const TfLiteIntArray& shape,
 }

 TfLiteStatus ResizeOutput(TfLiteContext* context, TfLiteNode* node) {
-  const TfLiteTensor* input = GetInput(context, node, kInputTensor);
-  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
-  const TfLiteTensor* multipliers = GetInput(context, node, kInputMultipliers);
+  const TfLiteTensor* input;
+  TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, kInputTensor, &input));
+  TfLiteTensor* output;
+  TF_LITE_ENSURE_OK(context,
+                    GetOutputSafe(context, node, kOutputTensor, &output));
+  const TfLiteTensor* multipliers;
+  TF_LITE_ENSURE_OK(
+      context, GetInputSafe(context, node, kInputMultipliers, &multipliers));

   const int num_dimensions = NumDimensions(input);
   const int num_multipliers = NumElements(multipliers);
@@ -208,12 +213,17 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   TF_LITE_ENSURE_EQ(context, NumInputs(node), 2);
   TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);

-  const TfLiteTensor* input = GetInput(context, node, kInputTensor);
+  const TfLiteTensor* input;
+  TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, kInputTensor, &input));

-  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+  TfLiteTensor* output;
+  TF_LITE_ENSURE_OK(context,
+                    GetOutputSafe(context, node, kOutputTensor, &output));
   TF_LITE_ENSURE_TYPES_EQ(context, input->type, output->type);

-  const TfLiteTensor* multipliers = GetInput(context, node, kInputMultipliers);
+  const TfLiteTensor* multipliers;
+  TF_LITE_ENSURE_OK(
+      context, GetInputSafe(context, node, kInputMultipliers, &multipliers));
   // Only int32 and int64 multipliers type is supported.
   if (multipliers->type != kTfLiteInt32 && multipliers->type != kTfLiteInt64) {
     context->ReportError(context,
@@ -231,9 +241,14 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
 }

 TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
-  const TfLiteTensor* input = GetInput(context, node, kInputTensor);
-  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
-  const TfLiteTensor* multipliers = GetInput(context, node, kInputMultipliers);
+  const TfLiteTensor* input;
+  TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, kInputTensor, &input));
+  TfLiteTensor* output;
+  TF_LITE_ENSURE_OK(context,
+                    GetOutputSafe(context, node, kOutputTensor, &output));
+  const TfLiteTensor* multipliers;
+  TF_LITE_ENSURE_OK(
+      context, GetInputSafe(context, node, kInputMultipliers, &multipliers));

   if (IsDynamicTensor(output)) {
     TF_LITE_ENSURE_OK(context, ResizeOutput(context, node));


[tflite]: Insert `nullptr` checks when obtaining tensors.

As part of ongoing refactoring, `tflite::GetInput`, `tflite::GetOutput`, `tflite::GetTemporary` and `tflite::GetIntermediates` will return `nullptr` in some cases. Hence, we insert the `nullptr` checks on all usages.

We also insert `nullptr` checks on usages of `tflite::GetVariableInput` and `tflite::GetOptionalInputTensor` but only in the cases where there is no obvious check that `nullptr` is acceptable (that is, we only insert the check for the output of these two functions if the tensor is accessed as if it is always not `nullptr`).

PiperOrigin-RevId: 332518902
Change-Id: I92eb164a6101ac3cca66090061a9b56a97288236

@@ -601,7 +601,8 @@ TfLiteStatus SimpleStatefulOp::Prepare(TfLiteContext* context,
   OpData* data = reinterpret_cast<OpData*>(node->user_data);

   // Make sure that the input is in uint8_t with at least 1 data entry.
-  const TfLiteTensor* input = tflite::GetInput(context, node, kInputTensor);
+  const TfLiteTensor* input;
+  TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, kInputTensor, &input));
   if (input->type != kTfLiteUInt8) return kTfLiteError;
   if (NumElements(input->dims) == 0) return kTfLiteError;

@@ -622,7 +623,8 @@ TfLiteStatus SimpleStatefulOp::Invoke(TfLiteContext* context,
   OpData* data = reinterpret_cast<OpData*>(node->user_data);
   *data->invoke_count += 1;

-  const TfLiteTensor* input = GetInput(context, node, kInputTensor);
+  const TfLiteTensor* input;
+  TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, kInputTensor, &input));
   const uint8_t* input_data = GetTensorData<uint8_t>(input);
   int size = NumElements(input->dims);

@@ -641,9 +643,13 @@ TfLiteStatus SimpleStatefulOp::Invoke(TfLiteContext* context,
     }
   }

-  TfLiteTensor* median = GetOutput(context, node, kMedianTensor);
+  TfLiteTensor* median;
+  TF_LITE_ENSURE_OK(context,
+                    GetOutputSafe(context, node, kMedianTensor, &median));
   uint8_t* median_data = GetTensorData<uint8_t>(median);
-  TfLiteTensor* invoke_count = GetOutput(context, node, kInvokeCount);
+  TfLiteTensor* invoke_count;
+  TF_LITE_ENSURE_OK(context,
+                    GetOutputSafe(context, node, kInvokeCount, &invoke_count));
   int32_t* invoke_count_data = GetTensorData<int32_t>(invoke_count);

   median_data[0] = sorting_buffer[size / 2];
@@ -681,11 +687,14 @@ TfLiteStatus MockCustom::Prepare(TfLiteContext* context, TfLiteNode* node) {
 }

 TfLiteStatus MockCustom::Invoke(TfLiteContext* context, TfLiteNode* node) {
-  const TfLiteTensor* input = tflite::GetInput(context, node, 0);
+  const TfLiteTensor* input;
+  TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, 0, &input));
   const int32_t* input_data = input->data.i32;
-  const TfLiteTensor* weight = tflite::GetInput(context, node, 1);
+  const TfLiteTensor* weight;
+  TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, 1, &weight));
   const uint8_t* weight_data = weight->data.uint8;
-  TfLiteTensor* output = GetOutput(context, node, 0);
+  TfLiteTensor* output;
+  TF_LITE_ENSURE_OK(context, GetOutputSafe(context, node, 0, &output));
   int32_t* output_data = output->data.i32;
   output_data[0] =
       0;  // Catch output tensor sharing memory with an input tensor


[tflite]: Insert `nullptr` checks when obtaining tensors.

As part of ongoing refactoring, `tflite::GetInput`, `tflite::GetOutput`, `tflite::GetTemporary` and `tflite::GetIntermediates` will return `nullptr` in some cases. Hence, we insert the `nullptr` checks on all usages.

We also insert `nullptr` checks on usages of `tflite::GetVariableInput` and `tflite::GetOptionalInputTensor` but only in the cases where there is no obvious check that `nullptr` is acceptable (that is, we only insert the check for the output of these two functions if the tensor is accessed as if it is always not `nullptr`).

PiperOrigin-RevId: 332517854
Change-Id: Ic27221dd1f0fbe302f311c2fe5a846ed8ff02016

@@ -621,8 +621,10 @@ TfLiteRegistration GetPassthroughOpRegistration() {
   reg.prepare = [](TfLiteContext* context, TfLiteNode* node) {
     auto* first_new_tensor = static_cast<int*>(node->user_data);

-    const TfLiteTensor* tensor0 = GetInput(context, node, 0);
-    TfLiteTensor* tensor1 = GetOutput(context, node, 0);
+    const TfLiteTensor* tensor0;
+    TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, 0, &tensor0));
+    TfLiteTensor* tensor1;
+    TF_LITE_ENSURE_OK(context, GetOutputSafe(context, node, 0, &tensor1));

     TfLiteIntArray* newSize = TfLiteIntArrayCopy(tensor0->dims);
     TF_LITE_ENSURE_STATUS(context->ResizeTensor(context, tensor1, newSize));
@@ -646,7 +648,8 @@ TfLiteRegistration GetPassthroughOpRegistration() {
     return kTfLiteOk;
   };
   reg.invoke = [](TfLiteContext* context, TfLiteNode* node) {
-    const TfLiteTensor* a0 = GetInput(context, node, 0);
+    const TfLiteTensor* a0;
+    TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, 0, &a0));

     auto populate = [&](int id) {
       TfLiteTensor* t = &context->tensors[id];
@@ -780,8 +783,10 @@ TEST(BasicInterpreter, ThreeStepAllocate) {
   // String-in String-out node.
   TfLiteRegistration reg_copy = {nullptr, nullptr, nullptr, nullptr};
   reg_copy.invoke = [](TfLiteContext* context, TfLiteNode* node) {
-    const TfLiteTensor* input = GetInput(context, node, 0);
-    TfLiteTensor* output = GetOutput(context, node, 0);
+    const TfLiteTensor* input;
+    TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, 0, &input));
+    TfLiteTensor* output;
+    TF_LITE_ENSURE_OK(context, GetOutputSafe(context, node, 0, &output));
     DynamicBuffer buf;
     StringRef str_ref = GetString(input, 0);
     buf.AddString(str_ref);
@@ -792,14 +797,17 @@ TEST(BasicInterpreter, ThreeStepAllocate) {
   // String-in Int-out node.
   TfLiteRegistration reg_len = {nullptr, nullptr, nullptr, nullptr};
   reg_len.prepare = [](TfLiteContext* context, TfLiteNode* node) {
-    TfLiteTensor* output = GetOutput(context, node, 0);
+    TfLiteTensor* output;
+    TF_LITE_ENSURE_OK(context, GetOutputSafe(context, node, 0, &output));
     TfLiteIntArray* outputSize = TfLiteIntArrayCreate(1);
     outputSize->data[0] = 1;
     return context->ResizeTensor(context, output, outputSize);
   };
   reg_len.invoke = [](TfLiteContext* context, TfLiteNode* node) {
-    const TfLiteTensor* a0 = GetInput(context, node, 0);
-    TfLiteTensor* a1 = GetOutput(context, node, 0);
+    const TfLiteTensor* a0;
+    TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, 0, &a0));
+    TfLiteTensor* a1;
+    TF_LITE_ENSURE_OK(context, GetOutputSafe(context, node, 0, &a1));
     a1->data.i32[0] = a0->bytes;
     return kTfLiteOk;
   };
@@ -848,14 +856,18 @@ TEST(BasicInterpreter, AllocateTwice) {

   TfLiteRegistration reg = {nullptr, nullptr, nullptr, nullptr};
   reg.prepare = [](TfLiteContext* context, TfLiteNode* node) {
-    const TfLiteTensor* tensor0 = GetInput(context, node, 0);
-    TfLiteTensor* tensor1 = GetOutput(context, node, 0);
+    const TfLiteTensor* tensor0;
+    TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, 0, &tensor0));
+    TfLiteTensor* tensor1;
+    TF_LITE_ENSURE_OK(context, GetOutputSafe(context, node, 0, &tensor1));
     TfLiteIntArray* newSize = TfLiteIntArrayCopy(tensor0->dims);
     return context->ResizeTensor(context, tensor1, newSize);
   };
   reg.invoke = [](TfLiteContext* context, TfLiteNode* node) {
-    const TfLiteTensor* a0 = GetInput(context, node, 0);
-    TfLiteTensor* a1 = GetOutput(context, node, 0);
+    const TfLiteTensor* a0;
+    TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, 0, &a0));
+    TfLiteTensor* a1;
+    TF_LITE_ENSURE_OK(context, GetOutputSafe(context, node, 0, &a1));
     int num = a0->dims->data[0];
     for (int i = 0; i < num; i++) {
       a1->data.f[i] = a0->data.f[i];
@@ -1205,8 +1217,10 @@ class TestExecutionPlan : public ::testing::Test {

     reg.prepare = [](TfLiteContext* context, TfLiteNode* node) {
       // Set output size to input size
-      const TfLiteTensor* tensor0 = GetInput(context, node, 0);
-      TfLiteTensor* tensor1 = GetOutput(context, node, 0);
+      const TfLiteTensor* tensor0;
+      TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, 0, &tensor0));
+      TfLiteTensor* tensor1;
+      TF_LITE_ENSURE_OK(context, GetOutputSafe(context, node, 0, &tensor1));
       TfLiteIntArray* newSize = TfLiteIntArrayCopy(tensor0->dims);
       return context->ResizeTensor(context, tensor1, newSize);
     };
@@ -1215,8 +1229,10 @@ class TestExecutionPlan : public ::testing::Test {
       CallReporting* call_reporting =
           static_cast<CallReporting*>(node->builtin_data);
       // Copy input data to output data.
-      const TfLiteTensor* a0 = GetInput(context, node, 0);
-      TfLiteTensor* a1 = GetOutput(context, node, 0);
+      const TfLiteTensor* a0;
+      TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, 0, &a0));
+      TfLiteTensor* a1;
+      TF_LITE_ENSURE_OK(context, GetOutputSafe(context, node, 0, &a1));
       int num = a0->dims->data[0];
       for (int i = 0; i < num; i++) {
         a1->data.f[i] = a0->data.f[i];
@@ -1403,8 +1419,10 @@ class CancellationTest : public ::testing::Test {
     // Set output size to the input size in CancelOp::Prepare(). Code exists to
     // have a framework in Prepare. The input and output tensors are not used.
     reg.prepare = [](TfLiteContext* context, TfLiteNode* node) {
-      const TfLiteTensor* in_tensor = GetInput(context, node, 0);
-      TfLiteTensor* out_tensor = GetOutput(context, node, 0);
+      const TfLiteTensor* in_tensor;
+      TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, 0, &in_tensor));
+      TfLiteTensor* out_tensor;
+      TF_LITE_ENSURE_OK(context, GetOutputSafe(context, node, 0, &out_tensor));
       TfLiteIntArray* new_size = TfLiteIntArrayCopy(in_tensor->dims);
       return context->ResizeTensor(context, out_tensor, new_size);
     };
@@ -1423,8 +1441,10 @@ class CancellationTest : public ::testing::Test {
     // Set output size to the input size in OkOp::Prepare(). Code exists to have
     // a framework in Prepare. The input and output tensors are not used.
     reg.prepare = [](TfLiteContext* context, TfLiteNode* node) {
-      const TfLiteTensor* in_tensor = GetInput(context, node, 0);
-      TfLiteTensor* out_tensor = GetOutput(context, node, 0);
+      const TfLiteTensor* in_tensor;
+      TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, 0, &in_tensor));
+      TfLiteTensor* out_tensor;
+      TF_LITE_ENSURE_OK(context, GetOutputSafe(context, node, 0, &out_tensor));
       TfLiteIntArray* new_size = TfLiteIntArrayCopy(in_tensor->dims);
       return context->ResizeTensor(context, out_tensor, new_size);
     };


[tflite] Make `GetOptionalInputTensor` the same as `GetInput`.

With the previous change, there is no more need for two separate APIs. We would deprecate `GetOptionalInputTensor` in the future.

PiperOrigin-RevId: 332513386
Change-Id: Id7110271c25ebd6126ad8c82a493e37e0e0756b3

@@ -75,12 +75,7 @@ TfLiteTensor* GetOutput(TfLiteContext* context, const TfLiteNode* node,

 const TfLiteTensor* GetOptionalInputTensor(const TfLiteContext* context,
                                            const TfLiteNode* node, int index) {
-  const bool use_tensor = index < node->inputs->size &&
-                          node->inputs->data[index] != kTfLiteOptionalTensor;
-  if (use_tensor) {
-    return GetMutableInput(context, node, index);
-  }
-  return nullptr;
+  return GetInput(context, node, index);
 }

 // Per-axis


[tflite]: Insert `nullptr` checks when obtaining tensors.

As part of ongoing refactoring, `tflite::GetInput`, `tflite::GetOutput`, `tflite::GetTemporary` and `tflite::GetIntermediates` will return `nullptr` in some cases. Hence, we insert the `nullptr` checks on all usages.

We also insert `nullptr` checks on usages of `tflite::GetVariableInput` and `tflite::GetOptionalInputTensor` but only in the cases where there is no obvious check that `nullptr` is acceptable (that is, we only insert the check for the output of these two functions if the tensor is accessed as if it is always not `nullptr`).

PiperOrigin-RevId: 332520146
Change-Id: I405d986cfc653aaafcfdf4162c0acbd46220b921

@@ -82,10 +82,13 @@ TfLiteStatus CalculateOpData(TfLiteContext* context, TfLiteNode* node,
   // parameters set. This is usually done during quantized training.
   if (data_type != kTfLiteFloat32) {
     const TfLiteTensor* input = GetInput(context, node, kInputTensor);
+    TF_LITE_ENSURE(context, input != nullptr);
     const TfLiteTensor* filter = GetInput(context, node, kFilterTensor);
+    TF_LITE_ENSURE(context, filter != nullptr);
     const TfLiteTensor* bias =
         GetOptionalInputTensor(context, node, kBiasTensor);
     TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+    TF_LITE_ENSURE(context, output != nullptr);
     int num_channels = filter->dims->data[kDepthwiseConvQuantizedDimension];

     return tflite::PopulateConvolutionQuantizationParams(
@@ -114,8 +117,11 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   OpData* data = static_cast<OpData*>(node->user_data);

   TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+  TF_LITE_ENSURE(context, output != nullptr);
   const TfLiteTensor* input = GetInput(context, node, kInputTensor);
+  TF_LITE_ENSURE(context, input != nullptr);
   const TfLiteTensor* filter = GetInput(context, node, kFilterTensor);
+  TF_LITE_ENSURE(context, filter != nullptr);

   const TfLiteType data_type = input->type;
   int width = SizeOfDimension(input, 2);


@@ -95,8 +95,11 @@ TfLiteStatus PreluPrepare(TfLiteContext* context, TfLiteNode* node) {
   PreluParams* params = static_cast<PreluParams*>(node->user_data);

   const TfLiteTensor* input = GetInput(context, node, 0);
+  TF_LITE_ENSURE(context, input != nullptr);
   const TfLiteTensor* alpha = GetInput(context, node, 1);
+  TF_LITE_ENSURE(context, alpha != nullptr);
   TfLiteTensor* output = GetOutput(context, node, 0);
+  TF_LITE_ENSURE(context, output != nullptr);

   return CalculatePreluParams(input, alpha, output, params);
 }


[tflite] Validate segment ids for segment_sum.

Segment identifiers in segment_sum should be in a 1-D tensor of same size as the first dimension of the input. The values of the tensor should be integers from {0, 1, 2, ... k-1}, where k is the first dimension of the input. The segment identifiers must not contain jumps and must be increasing.

See https://www.tensorflow.org/api_docs/python/tf/math#Segmentation as the source for these constraints.

PiperOrigin-RevId: 332510942
Change-Id: I898beaba00642c918bcd4b4d4ce893ebb190d869

@@ -34,11 +34,24 @@ TfLiteStatus ResizeOutputTensor(TfLiteContext* context,
                                 const TfLiteTensor* data,
                                 const TfLiteTensor* segment_ids,
                                 TfLiteTensor* output) {
-  int max_index = -1;
+  // Segment ids should be of same cardinality as first input dimension and they
+  // should be increasing by at most 1, from 0 (e.g., [0, 0, 1, 2, 3] is valid)
   const int segment_id_size = segment_ids->dims->data[0];
-  if (segment_id_size > 0) {
-    max_index = segment_ids->data.i32[segment_id_size - 1];
+  TF_LITE_ENSURE_EQ(context, segment_id_size, data->dims->data[0]);
+  int previous_segment_id = -1;
+  for (int i = 0; i < segment_id_size; i++) {
+    const int current_segment_id = GetTensorData<int32_t>(segment_ids)[i];
+    if (i == 0) {
+      TF_LITE_ENSURE_EQ(context, current_segment_id, 0);
+    } else {
+      int delta = current_segment_id - previous_segment_id;
+      TF_LITE_ENSURE(context, delta == 0 || delta == 1);
+    }
+    previous_segment_id = current_segment_id;
   }
+
+  const int max_index = previous_segment_id;
+
   const int data_rank = NumDimensions(data);
   TfLiteIntArray* output_shape = TfLiteIntArrayCreate(NumDimensions(data));
   output_shape->data[0] = max_index + 1;


Validate `data_splits` for `tf.StringNGrams`.

Without validation, we can cause a heap buffer overflow which results in data leakage and/or segfaults.

PiperOrigin-RevId: 332543478
Change-Id: Iee5bda24497a195d09d122355502480830b1b317

@@ -19,6 +19,7 @@ limitations under the License.
 #include "absl/strings/ascii.h"
 #include "absl/strings/str_cat.h"
 #include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/platform/errors.h"

 namespace tensorflow {
 namespace text {
@@ -60,6 +61,18 @@ class StringNGramsOp : public tensorflow::OpKernel {
     OP_REQUIRES_OK(context, context->input("data_splits", &splits));
     const auto& splits_vec = splits->flat<SPLITS_TYPE>();

+    // Validate that the splits are valid indices into data
+    const int input_data_size = data->flat<tstring>().size();
+    const int splits_vec_size = splits_vec.size();
+    for (int i = 0; i < splits_vec_size; ++i) {
+      bool valid_splits = splits_vec(i) >= 0;
+      valid_splits = valid_splits && (splits_vec(i) <= input_data_size);
+      OP_REQUIRES(
+          context, valid_splits,
+          errors::InvalidArgument("Invalid split value ", splits_vec(i),
+                                  ", must be in [0,", input_data_size, "]"));
+    }
+
     int num_batch_items = splits_vec.size() - 1;
     tensorflow::Tensor* ngrams_splits;
     OP_REQUIRES_OK(


Prevent integer truncation from 64 to 32 bits.

The `tensorflow::Shard` functions last argument must be a 2 argument function where both arguments are `int64` (`long long`, 64 bits). However, there are usages where code passes in a function where arguments are `int` or `int32` (32 bits). In these cases, it is possible that the integer truncation would later cause a segfault or other unexpected behavior.

PiperOrigin-RevId: 332560414
Change-Id: Ief649406babc8d4f60b3e7a9d573cbcc5ce5b767

@@ -193,7 +193,8 @@ struct LaunchBatchBandedTriangularSolve {

     Shard(worker_threads.num_threads, worker_threads.workers, batch_size,
           cost_per_unit,
-          [&in_x, &in_y, adjoint, lower, &bcast, out](int start, int limit) {
+          [&in_x, &in_y, adjoint, lower, &bcast, out](int64 start,
+                                                      int64 limit) {
             SequentialBandedTriangularSolveKernel<Scalar>::Run(
                 in_x, in_y, lower, adjoint, bcast, out, start, limit);
           });


Fix multiple vulnerabilities in `tf.raw_ops.*CountSparseOutput`.

Also add tests for these API points, both for the happy paths and for the vulnerable ones.

PiperOrigin-RevId: 332563222
Change-Id: Ib3b52116a83a134c2e742a7c66e5e956db8fba05

@@ -178,10 +178,30 @@ class SparseCount : public OpKernel {
     const Tensor& weights = context->input(3);
     bool use_weights = weights.NumElements() > 0;

+    OP_REQUIRES(context, TensorShapeUtils::IsMatrix(indices.shape()),
+                errors::InvalidArgument(
+                    "Input indices must be a 2-dimensional tensor. Got: ",
+                    indices.shape().DebugString()));
+
+    if (use_weights) {
+      OP_REQUIRES(
+          context, weights.shape() == values.shape(),
+          errors::InvalidArgument(
+              "Weights and values must have the same shape. Weight shape: ",
+              weights.shape().DebugString(),
+              "; values shape: ", values.shape().DebugString()));
+    }
+
     bool is_1d = shape.NumElements() == 1;
     int num_batches = is_1d ? 1 : shape.flat<int64>()(0);
     int num_values = values.NumElements();

+    OP_REQUIRES(context, num_values == indices.shape().dim_size(0),
+                errors::InvalidArgument(
+                    "Number of values must match first dimension of indices.",
+                    "Got ", num_values,
+                    " values, indices shape: ", indices.shape().DebugString()));
+
     const auto indices_values = indices.matrix<int64>();
     const auto values_values = values.flat<T>();
     const auto weight_values = weights.flat<W>();
@@ -235,12 +255,33 @@ class RaggedCount : public OpKernel {
     bool use_weights = weights.NumElements() > 0;
     bool is_1d = false;

+    if (use_weights) {
+      OP_REQUIRES(
+          context, weights.shape() == values.shape(),
+          errors::InvalidArgument(
+              "Weights and values must have the same shape. Weight shape: ",
+              weights.shape().DebugString(),
+              "; values shape: ", values.shape().DebugString()));
+    }
+
     const auto splits_values = splits.flat<int64>();
     const auto values_values = values.flat<T>();
     const auto weight_values = weights.flat<W>();
     int num_batches = splits.NumElements() - 1;
     int num_values = values.NumElements();

+    OP_REQUIRES(
+        context, num_batches > 0,
+        errors::InvalidArgument(
+            "Must provide at least 2 elements for the splits argument"));
+    OP_REQUIRES(context, splits_values(0) == 0,
+                errors::InvalidArgument("Splits must start with 0, not with ",
+                                        splits_values(0)));
+    OP_REQUIRES(context, splits_values(num_batches) == num_values,
+                errors::InvalidArgument(
+                    "Splits must end with the number of values, got ",
+                    splits_values(num_batches), " instead of ", num_values));
+
     auto per_batch_counts = BatchedMap<W>(num_batches);
     T max_value = 0;
     int batch_idx = 0;


Fix heap buffer overflow in `tf.raw_ops.SparseFillEmptyRowsGrad`.

Also add tests as they were lacking

PiperOrigin-RevId: 332566071
Change-Id: I44277578e26ff5fb3fdb0dcbba6e91b2ec3e7859

@@ -236,6 +236,9 @@ class SparseFillEmptyRowsGradOp : public OpKernel {
         context, TensorShapeUtils::IsVector(reverse_index_map_t->shape()),
         errors::InvalidArgument("reverse_index_map must be a vector, saw: ",
                                 reverse_index_map_t->shape().DebugString()));
+    OP_REQUIRES(context, TensorShapeUtils::IsVector(grad_values_t->shape()),
+                errors::InvalidArgument("grad_values must be a vector, saw: ",
+                                        grad_values_t->shape().DebugString()));

     const auto reverse_index_map = reverse_index_map_t->vec<int64>();
     const auto grad_values = grad_values_t->vec<T>();
@@ -264,8 +267,13 @@ class SparseFillEmptyRowsGradOp : public OpKernel {
       // Locate the index of the output of the forward prop associated
       // with this location in the input of the forward prop.  Copy
       // the gradient into it.  Mark it as visited.
-      d_values(i) = grad_values(reverse_index_map(i));
-      visited(reverse_index_map(i)) = true;
+      int64 reverse_index = reverse_index_map(i);
+      OP_REQUIRES(
+          context, 0 <= reverse_index && reverse_index < N_full,
+          errors::InvalidArgument("Elements in reverse index must be in [0, ",
+                                  N_full, ") but got ", reverse_index));
+      d_values(i) = grad_values(reverse_index);
+      visited(reverse_index) = true;
     }
     for (int j = 0; j < N_full; ++j) {
       // The default value gradient gets the accumulated remainder of


Fix multiple vulnerabilities in `tf.experimental.dlpack.to_dlpack`.

We have a use after free caused by memory coruption, a segmentation fault caused by memory corruption, several memory leaks and an undefined behavior when taking the reference of a nullptr.

PiperOrigin-RevId: 332568894
Change-Id: Ife0fc05e103b35325094ae5d822ee5fdea764572

@@ -1358,9 +1358,16 @@ PYBIND11_MODULE(_pywrap_tfe, m) {
   // DLPack functions
   m.def("TFE_ToDlpackCapsule", [](py::handle& o) {
     PyObject* eager_tensor_pyobject_ptr = o.ptr();
-    TFE_TensorHandle* thandle = EagerTensor_Handle(eager_tensor_pyobject_ptr);
     tensorflow::Safe_TF_StatusPtr status =
         tensorflow::make_safe(TF_NewStatus());
+
+    if (!EagerTensor_CheckExact(eager_tensor_pyobject_ptr)) {
+      status->status = tensorflow::errors::InvalidArgument(
+          "The argument to `to_dlpack` must be a TF tensor, not Python object");
+      tensorflow::MaybeRaiseRegisteredFromTFStatus(status.get());
+    }
+
+    TFE_TensorHandle* thandle = EagerTensor_Handle(eager_tensor_pyobject_ptr);
     void* dlm_ptr = tensorflow::TFE_HandleToDLPack(thandle, status.get());
     tensorflow::MaybeRaiseRegisteredFromTFStatus(status.get());


Add the new implementation of the silence module.

This module has significant improvements on the previous one:
 * Flags which actually make sense.
 * Better compatibility with other SILENCE implementations.
 * Support for blocking CTCPs.
 * Support for blocking TAGMSGs.

@@ -0,0 +1,438 @@
+/*
+ * InspIRCd -- Internet Relay Chat Daemon
+ *
+ *   Copyright (C) 2019 Peter Powell <petpow@saberuk.com>
+ *
+ * This file is part of InspIRCd.  InspIRCd is free software: you can
+ * redistribute it and/or modify it under the terms of the GNU General Public
+ * License as published by the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#include "inspircd.h"
+#include "modules/ctctags.h"
+
+enum
+{
+	// From ircu?
+	RPL_SILELIST = 271,
+	RPL_ENDOFSILELIST = 272,
+	ERR_SILELISTFULL = 511,
+
+	// InspIRCd-specific.
+	ERR_SILENCE = 952
+};
+
+class SilenceEntry
+{
+ public:
+	enum SilenceFlags
+	{
+		// Does nothing; for internal use only.
+		SF_NONE = 0,
+
+		// Exclude users who match this flags ("x").
+		SF_EXEMPT = 1,
+
+		// 2, 4, 8, 16 are reserved for future use.
+
+		// Matches a NOTICE targeted at a channel ("n").
+		SF_NOTICE_CHANNEL = 32,
+
+		// Matches a NOTICE targeted at a user ("N").
+		SF_NOTICE_USER = 64,
+
+		// Matches a PRIVMSG targeted at a channel ("p").
+		SF_PRIVMSG_CHANNEL = 128,
+
+		// Matches a PRIVMSG targeted at a user ("P").
+		SF_PRIVMSG_USER = 256,
+
+		// Matches a TAGMSG targeted at a channel ("t").
+		SF_TAGMSG_CHANNEL = 512,
+
+		// Matches a TAGMSG targeted at a user ("T").
+		SF_TAGMSG_USER = 1024,
+
+		// Matches a CTCP targeted at a channel ("c").
+		SF_CTCP_CHANNEL = 2048,
+
+		// Matches a CTCP targeted at a user ("C").
+		SF_CTCP_USER = 4096,
+
+		// Matches an invite to a channel ("i").
+		SF_INVITE = 8192,
+
+		// The default if no flags have been specified.
+		SF_DEFAULT = SF_NOTICE_CHANNEL | SF_NOTICE_USER | SF_PRIVMSG_CHANNEL | SF_PRIVMSG_USER | SF_TAGMSG_CHANNEL |
+			SF_TAGMSG_USER | SF_CTCP_CHANNEL | SF_CTCP_USER | SF_INVITE
+	};
+
+	// The flags that this mask is silenced for.
+	uint32_t flags;
+
+	// The mask which is silenced (e.g. *!*@example.com).
+	std::string mask;
+
+	SilenceEntry(uint32_t Flags, const std::string& Mask)
+		: flags(Flags)
+		, mask(Mask)
+	{
+	}
+
+	bool operator <(const SilenceEntry& other) const
+	{
+		if (flags & SF_EXEMPT && other.flags & ~SF_EXEMPT)
+			return true;
+		if (other.flags & SF_EXEMPT && flags & ~SF_EXEMPT)
+			return false;
+		if (flags < other.flags)
+			return true;
+		if (other.flags < flags)
+			return false;
+		return mask < other.mask;
+	}
+
+	// Converts a flag list to a bitmask.
+	static bool FlagsToBits(const std::string& flags, uint32_t& out)
+	{
+		out = SF_NONE;
+		for (std::string::const_iterator flag = flags.begin(); flag != flags.end(); ++flag)
+		{
+			switch (*flag)
+			{
+				case 'C':
+					out |= SF_CTCP_USER;
+					break;
+				case 'c':
+					out |= SF_CTCP_CHANNEL;
+					break;
+				case 'd':
+					out |= SF_DEFAULT;
+					break;
+				case 'i':
+					out |= SF_INVITE;
+					break;
+				case 'N':
+					out |= SF_NOTICE_USER;
+					break;
+				case 'n':
+					out |= SF_NOTICE_CHANNEL;
+					break;
+				case 'P':
+					out |= SF_PRIVMSG_USER;
+					break;
+				case 'p':
+					out |= SF_PRIVMSG_CHANNEL;
+					break;
+				case 'T':
+					out |= SF_TAGMSG_USER;
+					break;
+				case 't':
+					out |= SF_TAGMSG_CHANNEL;
+					break;
+				case 'x':
+					out |= SF_EXEMPT;
+					break;
+				default:
+					out = SF_NONE;
+					return false;
+			}
+		}
+		return true;
+	}
+
+	// Converts a bitmask to a flag list.
+	static std::string BitsToFlags(uint32_t flags)
+	{
+		std::string out;
+		if (flags & SF_CTCP_USER)
+			out.push_back('C');
+		if (flags & SF_CTCP_CHANNEL)
+			out.push_back('c');
+		if (flags & SF_INVITE)
+			out.push_back('i');
+		if (flags & SF_NOTICE_USER)
+			out.push_back('N');
+		if (flags & SF_NOTICE_CHANNEL)
+			out.push_back('n');
+		if (flags & SF_PRIVMSG_USER)
+			out.push_back('P');
+		if (flags & SF_PRIVMSG_CHANNEL)
+			out.push_back('p');
+		if (flags & SF_TAGMSG_CHANNEL)
+			out.push_back('T');
+		if (flags & SF_TAGMSG_USER)
+			out.push_back('t');
+		if (flags & SF_EXEMPT)
+			out.push_back('x');
+		return out;
+	}
+};
+
+typedef insp::flat_set<SilenceEntry> SilenceList;
+
+class SilenceMessage : public ClientProtocol::Message
+{
+ public:
+	SilenceMessage(const std::string& mask, const std::string& flags)
+		: ClientProtocol::Message("SILENCE")
+	{
+		PushParam(mask);
+		PushParamRef(flags);
+	}
+};
+
+class CommandSilence : public SplitCommand
+{
+ private:
+	ClientProtocol::EventProvider msgprov;
+
+	CmdResult AddSilence(LocalUser* user, const std::string& mask, uint32_t flags)
+	{
+		SilenceList* list = ext.get(user);
+		if (list && list->size() > maxsilence)
+		{
+			user->WriteNumeric(ERR_SILELISTFULL, mask, SilenceEntry::BitsToFlags(flags), "Your silence list is full");
+			return CMD_FAILURE;
+		}
+		else if (!list)
+		{
+			// There is no list; create it.
+			list = new SilenceList();
+			ext.set(user, list);
+		}
+
+		if (!list->insert(SilenceEntry(flags, mask)).second)
+		{
+			user->WriteNumeric(ERR_SILENCE, mask, SilenceEntry::BitsToFlags(flags), "The silence entry you specified already exists");
+			return CMD_FAILURE;
+		}
+
+		SilenceMessage msg("+" + mask, SilenceEntry::BitsToFlags(flags));
+		user->Send(msgprov, msg);
+		return CMD_SUCCESS;
+	}
+
+	CmdResult RemoveSilence(LocalUser* user, const std::string& mask, uint32_t flags)
+	{
+		SilenceList* list = ext.get(user);
+		if (list)
+		{
+			for (SilenceList::iterator iter = list->begin(); iter != list->end(); ++iter)
+			{
+				if (!irc::equals(iter->mask, mask) || iter->flags != flags)
+					continue;
+
+				list->erase(iter);
+				SilenceMessage msg("-" + mask, SilenceEntry::BitsToFlags(flags));
+				user->Send(msgprov, msg);
+				return CMD_SUCCESS;
+			}
+		}
+
+		user->WriteNumeric(ERR_SILENCE, mask, SilenceEntry::BitsToFlags(flags), "The silence entry you specified could not be found");
+		return CMD_FAILURE;
+	}
+
+	CmdResult ShowSilenceList(LocalUser* user)
+	{
+		SilenceList* list = ext.get(user);
+		if (list)
+		{
+			for (SilenceList::const_iterator iter = list->begin(); iter != list->end(); ++iter)
+			{
+				user->WriteNumeric(RPL_SILELIST, iter->mask, SilenceEntry::BitsToFlags(iter->flags));
+			}
+		}
+		user->WriteNumeric(RPL_ENDOFSILELIST, "End of silence list");
+		return CMD_SUCCESS;
+	}
+
+ public:
+	SimpleExtItem<SilenceList> ext;
+	unsigned int maxsilence;
+
+	CommandSilence(Module* Creator)
+		: SplitCommand(Creator, "SILENCE")
+		, msgprov(Creator, "SILENCE")
+		, ext("silence_list", ExtensionItem::EXT_USER, Creator)
+	{
+		allow_empty_last_param = false;
+		syntax = "[(+|-)<mask> [CcdiNnPpTtx]]";
+	}
+
+	CmdResult HandleLocal(LocalUser* user, const Params& parameters) CXX11_OVERRIDE
+	{
+		if (parameters.empty())
+			return ShowSilenceList(user);
+
+		// If neither add nor remove are specified we default to add.
+		bool is_remove = parameters[0][0] == '-';
+
+		// If a prefix mask has been given then strip it and clean it up.
+		std::string mask = parameters[0];
+		if (mask[0] == '-' || mask[0] == '+')
+		{
+			mask.erase(0);
+			if (mask.empty())
+				mask.assign("*");
+			ModeParser::CleanMask(mask);
+		}
+
+		// If the user specified a flags then use that. Otherwise, default to blocking
+		// all CTCPs, invites, notices, privmsgs, and invites.
+		uint32_t flags = SilenceEntry::SF_DEFAULT;
+		if (parameters.size() > 1)
+		{
+			if (!SilenceEntry::FlagsToBits(parameters[1], flags))
+			{
+				user->WriteNumeric(ERR_SILENCE, mask, parameters[1], "You specified one or more invalid SILENCE flags");
+				return CMD_FAILURE;
+			}
+			else if (flags == SilenceEntry::SF_EXEMPT)
+			{
+				// The user specified "x" with no other flags which does not make sense; add the "d" flag.
+				flags |= SilenceEntry::SF_DEFAULT;
+			}
+		}
+
+		return is_remove ? RemoveSilence(user, mask, flags) : AddSilence(user, mask, flags);
+	}
+};
+
+class ModuleSilence
+	: public Module
+	, public CTCTags::EventListener
+{
+ private:
+	bool exemptuline;
+	CommandSilence cmd;
+
+	ModResult BuildChannelExempts(User* source, Channel* channel, SilenceEntry::SilenceFlags flag, CUList& exemptions)
+	{
+		const Channel::MemberMap& members = channel->GetUsers();
+		for (Channel::MemberMap::const_iterator member = members.begin(); member != members.end(); ++member)
+		{
+			if (!CanReceiveMessage(source, member->first, flag))
+				exemptions.insert(member->first);
+		}
+		return MOD_RES_PASSTHRU;
+	}
+
+	bool CanReceiveMessage(User* source, User* target, SilenceEntry::SilenceFlags flag)
+	{
+		// Servers handle their own clients.
+		if (!IS_LOCAL(target))
+			return true;
+
+		if (exemptuline && source->server->IsULine())
+			return true;
+
+		SilenceList* list = cmd.ext.get(target);
+		if (!list)
+			return true;
+
+		for (SilenceList::iterator iter = list->begin(); iter != list->end(); ++iter)
+		{
+			if (!(iter->flags & flag))
+				continue;
+
+			if (InspIRCd::Match(source->GetFullHost(), iter->mask))
+				return iter->flags & SilenceEntry::SF_EXEMPT;
+		}
+
+		return true;
+	}
+
+ public:
+	ModuleSilence()
+		: CTCTags::EventListener(this)
+		, cmd(this)
+	{
+	}
+
+	void ReadConfig(ConfigStatus& status) CXX11_OVERRIDE
+	{
+		ConfigTag* tag = ServerInstance->Config->ConfValue("silence");
+		exemptuline = tag->getBool("exemptuline", true);
+		cmd.maxsilence = tag->getUInt("maxentries", 32, 1);
+	}
+
+	void On005Numeric(std::map<std::string, std::string>& tokens) CXX11_OVERRIDE
+	{
+		tokens["ESILENCE"] = "CcdiNnPpsTtx";
+		tokens["SILENCE"] = ConvToStr(cmd.maxsilence);
+	}
+
+	ModResult OnUserPreInvite(User* source, User* dest, Channel* channel, time_t timeout) CXX11_OVERRIDE
+	{
+		return CanReceiveMessage(source, dest, SilenceEntry::SF_INVITE) ? MOD_RES_PASSTHRU : MOD_RES_DENY;
+	}
+
+	ModResult OnUserPreMessage(User* user, const MessageTarget& target, MessageDetails& details) CXX11_OVERRIDE
+	{
+		std::string ctcpname;
+		bool is_ctcp = details.IsCTCP(ctcpname) && !irc::equals(ctcpname, "ACTION");
+
+		SilenceEntry::SilenceFlags flag = SilenceEntry::SF_NONE;
+		if (target.type == MessageTarget::TYPE_CHANNEL)
+		{
+			if (is_ctcp)
+				flag = SilenceEntry::SF_CTCP_CHANNEL;
+			else if (details.type == MSG_NOTICE)
+				flag = SilenceEntry::SF_NOTICE_CHANNEL;
+			else if (details.type == MSG_PRIVMSG)
+				flag = SilenceEntry::SF_PRIVMSG_CHANNEL;
+
+			return BuildChannelExempts(user, target.Get<Channel>(), flag, details.exemptions);
+		}
+
+		if (target.type == MessageTarget::TYPE_USER)
+		{
+			if (is_ctcp)
+				flag = SilenceEntry::SF_CTCP_USER;
+			else if (details.type == MSG_NOTICE)
+				flag = SilenceEntry::SF_NOTICE_USER;
+			else if (details.type == MSG_PRIVMSG)
+				flag = SilenceEntry::SF_PRIVMSG_USER;
+
+			if (!CanReceiveMessage(user, target.Get<User>(), flag))
+			{
+				details.echo_original = true;
+				return MOD_RES_DENY;
+			}
+		}
+
+		return MOD_RES_PASSTHRU;
+	}
+
+	ModResult OnUserPreTagMessage(User* user, const MessageTarget& target, CTCTags::TagMessageDetails& details) CXX11_OVERRIDE
+	{
+		if (target.type == MessageTarget::TYPE_CHANNEL)
+			return BuildChannelExempts(user, target.Get<Channel>(), SilenceEntry::SF_TAGMSG_CHANNEL, details.exemptions);
+
+		if (target.type == MessageTarget::TYPE_USER && !CanReceiveMessage(user, target.Get<User>(), SilenceEntry::SF_TAGMSG_USER))
+		{
+			details.echo_original = true;
+			return MOD_RES_DENY;
+		}
+
+		return MOD_RES_PASSTHRU;
+	}
+
+	Version GetVersion() CXX11_OVERRIDE
+	{
+		return Version("Provides support for blocking users with the /SILENCE command", VF_OPTCOMMON | VF_VENDOR);
+	}
+};
+
+MODULE_INIT(ModuleSilence)


WindowsServiceControl: quote service binary path

Fix unquoted service path vulnerability.

Closes #657.

@@ -158,6 +158,8 @@ bool WindowsServiceControl::stop()

 bool WindowsServiceControl::install( const QString& filePath, const QString& displayName  )
 {
+	const auto binaryPath = QStringLiteral("\"%1\"").arg( QString( filePath ).replace( QLatin1Char('"'), QString() ) );
+
 	m_serviceHandle = CreateService(
 				m_serviceManager,		// SCManager database
 				WindowsCoreFunctions::toConstWCharArray( m_name ),	// name of service
@@ -167,7 +169,7 @@ bool WindowsServiceControl::install( const QString& filePath, const QString& dis
 				// service type
 				SERVICE_AUTO_START,	// start type
 				SERVICE_ERROR_NORMAL,	// error control type
-				WindowsCoreFunctions::toConstWCharArray( filePath ),		// service's binary
+				WindowsCoreFunctions::toConstWCharArray( binaryPath ),		// service's binary
 				nullptr,			// no load ordering group
 				nullptr,			// no tag identifier
 				L"Tcpip\0RpcSs\0\0",		// dependencies


Drop unreliable segments with weird offset/size.

And be more deliberate about limits of unreliable message/segment sizes.

@@ -234,7 +234,7 @@ int64 CSteamNetworkConnectionBase::SNP_SendMessage( CSteamNetworkingMessage *pSe
 	}

 	// Check if they try to send a really large message
-	if ( cbData > k_cbMaxUnreliableMsgSize && !( pSendMessage->m_nFlags & k_nSteamNetworkingSend_Reliable )  )
+	if ( cbData > k_cbMaxUnreliableMsgSizeSend && !( pSendMessage->m_nFlags & k_nSteamNetworkingSend_Reliable )  )
 	{
 		SpewWarningRateLimited( usecNow, "Trying to send a very large (%d bytes) unreliable message.  Sending as reliable instead.\n", cbData );
 		pSendMessage->m_nFlags |= k_nSteamNetworkingSend_Reliable;
@@ -577,11 +577,23 @@ bool CSteamNetworkConnectionBase::ProcessPlainTextDataChunk( int usecTimeSinceLa
 			// Decode size, locate segment data
 			//
 			READ_SEGMENT_DATA_SIZE( unreliable )
-			Assert( cbSegmentSize > 0 ); // !TEST! Bogus assert, zero byte messages are OK.  Remove after testing

-			// Receive the segment
-			bool bLastSegmentInMessage = ( nFrameType & 0x20 ) != 0;
-			SNP_ReceiveUnreliableSegment( nCurMsgNum, nOffset, pSegmentData, cbSegmentSize, bLastSegmentInMessage, usecNow );
+			// Check if offset+size indicates a message larger than what we support.  (Also,
+			// protect against malicious sender sending *extremely* large offset causing overflow.)
+			if ( (int64)nOffset + cbSegmentSize > k_cbMaxUnreliableMsgSizeRecv || cbSegmentSize > k_cbMaxUnreliableSegmentSizeRecv )
+			{
+
+				// Since this is unreliable data, we can just ignore the segment.
+				SpewWarningRateLimited( usecNow, "[%s] Ignoring unreliable segment with invalid offset %u size %d\n",
+					GetDescription(), nOffset, cbSegmentSize );
+			}
+			else
+			{
+
+				// Receive the segment
+				bool bLastSegmentInMessage = ( nFrameType & 0x20 ) != 0;
+				SNP_ReceiveUnreliableSegment( nCurMsgNum, nOffset, pSegmentData, cbSegmentSize, bLastSegmentInMessage, usecNow );
+			}
 		}
 		else if ( ( nFrameType & 0xe0 ) == 0x40 )
 		{


Validate that `DataFormat*` attributes form a permutation.

The `src_format` and `dst_format` attributes for the `DataFormatDimMap` and `DataFormatVecPermute` raw ops are supposed to determine a permutation. However, this was not validated and could result in unitialized memory accesses as well as writes outside of bounds and potential crashes.

While here, we also test that the format attributes have the needed length, add tests for all validation failure cases, remove unnecessary calls to `strings::StrCat`, and fix a few grammar errors.

This will be cherry-picked on the supported release branches.

PiperOrigin-RevId: 346135579
Change-Id: I1c76392382c89ad8f072d5bc93d70669851eb404

@@ -18,16 +18,52 @@ limitations under the License.
 #define EIGEN_USE_THREADS

 #include "tensorflow/core/kernels/data_format_ops.h"
+
+#include <map>
+
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/platform/errors.h"

 namespace tensorflow {

 typedef Eigen::ThreadPoolDevice CPUDevice;
 typedef Eigen::GpuDevice GPUDevice;

+// Ensure that `src` and `dst` define a valid permutation.
+// Ops defined in this file assume that user specifies a permutation via two
+// string attributes. This check validates that these attributes properly define
+// it to prevent security vulnerabilities.
+static bool IsValidPermutation(const std::string& src, const std::string& dst) {
+  if (src.size() != dst.size()) {
+    return false;
+  }
+
+  std::map<char, bool> characters;
+
+  // Every character in `src` must be present only once
+  for (const auto c : src) {
+    if (characters[c]) {
+      return false;
+    }
+    characters[c] = true;
+  }
+
+  // Every character in `dst` must show up in `src` exactly once
+  for (const auto c : dst) {
+    if (!characters[c]) {
+      return false;
+    }
+    characters[c] = false;
+  }
+
+  // At this point, characters[] has been switched to true and false exactly
+  // once for all character in `src` (and `dst`) so we have a valid permutation
+  return true;
+}
+
 template <typename Device, typename T>
 class DataFormatDimMapOp : public OpKernel {
  public:
@@ -38,15 +74,19 @@ class DataFormatDimMapOp : public OpKernel {
     string dst_format;
     OP_REQUIRES_OK(context, context->GetAttr("dst_format", &dst_format));
     OP_REQUIRES(context, src_format.size() == 4 || src_format.size() == 5,
-                errors::InvalidArgument(strings::StrCat(
-                    "Source format must of length 4 or 5, received "
+                errors::InvalidArgument(
+                    "Source format must be of length 4 or 5, received "
                     "src_format = ",
-                    src_format)));
+                    src_format));
+    OP_REQUIRES(context, dst_format.size() == 4 || dst_format.size() == 5,
+                errors::InvalidArgument("Destination format must be of length "
+                                        "4 or 5, received dst_format = ",
+                                        dst_format));
     OP_REQUIRES(
-        context, dst_format.size() == 4 || dst_format.size() == 5,
-        errors::InvalidArgument(strings::StrCat(
-            "Destination format must of length 4 or 5, received dst_format = ",
-            dst_format)));
+        context, IsValidPermutation(src_format, dst_format),
+        errors::InvalidArgument(
+            "Destination and source format must determine a permutation, got ",
+            src_format, " and ", dst_format));
     dst_idx_ = Tensor(DT_INT32, {static_cast<int64>(src_format.size())});
     for (int i = 0; i < src_format.size(); ++i) {
       for (int j = 0; j < dst_format.size(); ++j) {
@@ -78,8 +118,22 @@ class DataFormatVecPermuteOp : public OpKernel {
       : OpKernel(context) {
     string src_format;
     OP_REQUIRES_OK(context, context->GetAttr("src_format", &src_format));
+    OP_REQUIRES(context, src_format.size() == 4 || src_format.size() == 5,
+                errors::InvalidArgument(
+                    "Source format must be of length 4 or 5, received "
+                    "src_format = ",
+                    src_format));
     string dst_format;
     OP_REQUIRES_OK(context, context->GetAttr("dst_format", &dst_format));
+    OP_REQUIRES(context, dst_format.size() == 4 || dst_format.size() == 5,
+                errors::InvalidArgument("Destination format must be of length "
+                                        "4 or 5, received dst_format = ",
+                                        dst_format));
+    OP_REQUIRES(
+        context, IsValidPermutation(src_format, dst_format),
+        errors::InvalidArgument(
+            "Destination and source format must determine a permutation, got ",
+            src_format, " and ", dst_format));
     src_format_ = src_format;
     dst_format_ = dst_format;
   }
@@ -127,6 +181,10 @@ class DataFormatVecPermuteOp : public OpKernel {
       };
       keep_only_spatial_dimensions(&src_format_str);
       keep_only_spatial_dimensions(&dst_format_str);
+      OP_REQUIRES(context,
+                  src_format_str.size() == 2 && dst_format_str.size() == 2,
+                  errors::InvalidArgument(
+                      "Format specifier must contain H and W for 2D case"));
     }
     ComputeDstIndex(src_format_str, dst_format_str, input.dims(), &dst_idx);


Add negative caching tests and fixes. (#7361)

This adds test coverage for the negative caching feature and makes some
fixes as a result of the test's findings.

@@ -1038,7 +1038,7 @@ set_negative_caching_list(const char *name, RecDataT dtype, RecData data, HttpCo
   HttpStatusBitset set;
   // values from proxy.config.http.negative_caching_list
   if (0 == strcasecmp("proxy.config.http.negative_caching_list", name) && RECD_STRING == dtype && data.rec_string) {
-    // parse the list of status code
+    // parse the list of status codes
     ts::TextView status_list(data.rec_string, strlen(data.rec_string));
     auto is_sep{[](char c) { return isspace(c) || ',' == c || ';' == c; }};
     while (!status_list.ltrim_if(is_sep).empty()) {


@@ -682,6 +682,13 @@ void IMAPSession::connect(ErrorCode * pError)
             * pError = ErrorTLSNotAvailable;
             goto close;
         }
+
+        mIsCertificateValid = checkCertificate();
+        if (isCheckCertificateEnabled() && !mIsCertificateValid) {
+            * pError = ErrorCertificate;
+            goto close;
+        }
+
         break;

         case ConnectionTypeTLS:


jwt_authn: fix a bug where JWT with wrong issuer is allowed in allow_missing case (#15194)

[jwt] When allow_missing is used inside RequiresAny, the requests with JWT with wrong issuer are accepted. This is a bug, allow_missing should only allow requests without any JWT. This change fixed the above issue by preserving JwtUnknownIssuer in allow_missing case.

Signed-off-by: Wayne Zhang <qiwzhang@google.com>

@@ -197,7 +197,7 @@ TEST_F(SingleAllowMissingInOrListTest, BadJwt) {
 }

 TEST_F(SingleAllowMissingInOrListTest, MissingIssToken) {
-  EXPECT_CALL(mock_cb_, onComplete(Status::Ok));
+  EXPECT_CALL(mock_cb_, onComplete(Status::JwtUnknownIssuer));
   auto headers = Http::TestRequestHeaderMapImpl{{kExampleHeader, ES256WithoutIssToken}};
   context_ = Verifier::createContext(headers, parent_span_, &mock_cb_);
   verifier_->verify(context_);
@@ -471,6 +471,15 @@ TEST_F(AllowMissingInOrListTest, OtherGoodJwt) {
   EXPECT_THAT(headers, JwtOutputFailedOrIgnore(kOtherHeader));
 }

+TEST_F(AllowMissingInOrListTest, WrongIssuer) {
+  EXPECT_CALL(mock_cb_, onComplete(Status::JwtUnknownIssuer));
+  auto headers = Http::TestRequestHeaderMapImpl{{kExampleHeader, OtherGoodToken}};
+  context_ = Verifier::createContext(headers, parent_span_, &mock_cb_);
+  verifier_->verify(context_);
+  // x-other JWT should be ignored.
+  EXPECT_THAT(headers, JwtOutputFailedOrIgnore(kOtherHeader));
+}
+
 TEST_F(AllowMissingInOrListTest, BadAndGoodJwts) {
   EXPECT_CALL(mock_cb_, onComplete(Status::JwtVerificationFail));
   auto headers = Http::TestRequestHeaderMapImpl{{kExampleHeader, NonExistKidToken},
@@ -589,7 +598,7 @@ TEST_F(AllowMissingInAndOfOrListTest, TwoGoodJwts) {
 }

 TEST_F(AllowMissingInAndOfOrListTest, GoodAndBadJwts) {
-  EXPECT_CALL(mock_cb_, onComplete(Status::Ok));
+  EXPECT_CALL(mock_cb_, onComplete(Status::JwtUnknownIssuer));
   // Use the token with example.com issuer for x-other.
   auto headers =
       Http::TestRequestHeaderMapImpl{{kExampleHeader, GoodToken}, {kOtherHeader, GoodToken}};


security fixes

https://hhvm.com/blog/2021/02/25/security-update.html

@@ -1925,6 +1925,9 @@ String preg_quote(const String& str,

   /* Allocate enough memory so that even if each character
      is quoted, we won't run out of room */
+  static_assert(
+    (StringData::MaxSize * 4 + 1) < std::numeric_limits<int64_t>::max()
+  );
   String ret(4 * str.size() + 1, ReserveString);
   char* out_str = ret.mutableData();


@@ -454,7 +454,7 @@ ALWAYS_INLINE String serialize_impl(const Variant& value,
         lazyClassToStringHelper(value.toLazyClassVal());
       auto const size = str->size();
       if (size >= RuntimeOption::MaxSerializedStringSize) {
-        throw Exception("Size of serialized string (%d) exceeds max", size);
+        throw Exception("Size of serialized string (%ld) exceeds max", size);
       }
       StringBuffer sb;
       sb.append("s:");


@@ -195,7 +195,7 @@ struct String {
   }

 public:
-  const String& setSize(int len) {
+  const String& setSize(int64_t len) {
     assertx(m_str);
     m_str->setSize(len);
     return *this;
@@ -224,10 +224,10 @@ struct String {
   bool empty() const {
     return m_str ? m_str->empty() : true;
   }
-  int size() const {
+  int64_t size() const {
     return m_str ? m_str->size() : 0;
   }
-  int length() const {
+  int64_t length() const {
     return m_str ? m_str->size() : 0;
   }
   uint32_t capacity() const {
@@ -591,4 +591,3 @@ template<> class FormatValue<HPHP::StaticString> {
   const HPHP::StaticString& m_val;
 };
 }
-


@@ -656,7 +656,7 @@ Variant HHVM_FUNCTION(mcrypt_generic_init, const Resource& td,

   int key_size;
   if (key.size() > max_key_size) {
-    raise_warning("Key size too large; supplied length: %d, max: %d",
+    raise_warning("Key size too large; supplied length: %ld, max: %d",
                     key.size(), max_key_size);
     key_size = max_key_size;
   } else {
@@ -665,10 +665,10 @@ Variant HHVM_FUNCTION(mcrypt_generic_init, const Resource& td,
   memcpy(key_s, key.data(), key.size());

   if (iv.size() != iv_size) {
-    raise_warning("Iv size incorrect; supplied length: %d, needed: %d",
+    raise_warning("Iv size incorrect; supplied length: %ld, needed: %d",
                     iv.size(), iv_size);
   }
-  memcpy(iv_s, iv.data(), std::min(iv_size, iv.size()));
+  memcpy(iv_s, iv.data(), std::min<int64_t>(iv_size, iv.size()));

   mcrypt_generic_deinit(pm->m_td);
   int result = mcrypt_generic_init(pm->m_td, key_s, key_size, iv_s);


@@ -109,13 +109,15 @@ bool MemFile::closeImpl() {
 int64_t MemFile::readImpl(char *buffer, int64_t length) {
   assertx(m_len != -1);
   assertx(length > 0);
+  assertx(m_cursor >= 0);
   int64_t remaining = m_len - m_cursor;
   if (remaining < length) length = remaining;
   if (length > 0) {
     memcpy(buffer, (const void *)(m_data + m_cursor), length);
+    m_cursor += length;
+    return length;
   }
-  m_cursor += length;
-  return length;
+  return 0;
 }

 int MemFile::getc() {
@@ -126,7 +128,7 @@ int MemFile::getc() {
 bool MemFile::seek(int64_t offset, int whence /* = SEEK_SET */) {
   assertx(m_len != -1);
   if (whence == SEEK_CUR) {
-    if (offset > 0 && offset < bufferedLen()) {
+    if (offset >= 0 && offset < bufferedLen()) {
       setReadPosition(getReadPosition() + offset);
       setPosition(getPosition() + offset);
       return true;
@@ -139,10 +141,13 @@ bool MemFile::seek(int64_t offset, int whence /* = SEEK_SET */) {
   setWritePosition(0);
   setReadPosition(0);
   if (whence == SEEK_SET) {
+    if (offset < 0) return false;
     m_cursor = offset;
-  } else {
-    assertx(whence == SEEK_END);
+  } else if (whence == SEEK_END) {
+    if (m_len + offset < 0) return false;
     m_cursor = m_len + offset;
+  } else {
+    return false;
   }
   setPosition(m_cursor);
   return true;


@@ -464,7 +464,7 @@ Variant HHVM_FUNCTION(fwrite,
   CHECK_HANDLE(handle, f);
   int64_t ret = f->write(data, length);
   if (ret < 0) {
-    raise_notice("fwrite(): send of %d bytes failed with errno=%d %s",
+    raise_notice("fwrite(): send of %ld bytes failed with errno=%d %s",
                  data.size(), errno, folly::errnoStr(errno).c_str());
     ret = 0;
   }


security fixes

https://hhvm.com/blog/2020/11/12/security-update.html

@@ -496,6 +496,8 @@ struct RuntimeOption {
   static std::set<std::string> AdminPasswords;
   static std::set<std::string> HashedAdminPasswords;

+  static std::string AdminDumpPath;
+
   /*
    * Options related to reverse proxying. ProxyOriginRaw and ProxyPercentageRaw
    * may be mutated by background threads and should only be read or written


@@ -696,7 +696,10 @@ static char *BF_crypt(const char *key, const char *setting,
   }

   count = (BF_word)1 << ((setting[4] - '0') * 10 + (setting[5] - '0'));
-  if (count < min || BF_decode(data.binary.salt, &setting[7], 16)) {
+  if (count < min ||
+      BF_decode(data.binary.salt, &setting[7], 16) ||
+      ((unsigned int)(setting[7 + 22 - 1] - 0x20) >= 0x60)
+     ) {
     __set_errno(EINVAL);
     return NULL;
   }
@@ -914,4 +917,3 @@ char *_crypt_gensalt_blowfish_rn(const char *prefix, unsigned long count,
   return output;
 }
 #endif
-


@@ -2940,7 +2940,7 @@ static bool php_openssl_validate_iv(
   }

   if (piv.size() < iv_required_len) {
-    raise_warning("IV passed is only %d bytes long, cipher "
+    raise_warning("IV passed is only %ld bytes long, cipher "
                   "expects an IV of precisely %d bytes, padding with \\0",
                   piv.size(), iv_required_len);
     memcpy(iv_new, piv.data(), piv.size());
@@ -2949,7 +2949,7 @@ static bool php_openssl_validate_iv(
     return true;
   }

-  raise_warning("IV passed is %d bytes long which is longer than the %d "
+  raise_warning("IV passed is %ld bytes long which is longer than the %d "
                 "expected by selected cipher, truncating", piv.size(),
                 iv_required_len);
   memcpy(iv_new, piv.data(), iv_required_len);


Improve bound checking in WebPImage::doWriteMetadata()

@@ -145,7 +145,7 @@ namespace Exiv2 {
         DataBuf chunkId(WEBP_TAG_SIZE+1);
         chunkId.pData_ [WEBP_TAG_SIZE] = '\0';

-        io_->read(data, WEBP_TAG_SIZE * 3);
+        readOrThrow(*io_, data, WEBP_TAG_SIZE * 3, Exiv2::kerCorruptedMetadata);
         uint64_t filesize = Exiv2::getULong(data + WEBP_TAG_SIZE, littleEndian);

         /* Set up header */
@@ -185,20 +185,28 @@ namespace Exiv2 {
          case we have any exif or xmp data, also check
          for any chunks with alpha frame/layer set */
         while ( !io_->eof() && (uint64_t) io_->tell() < filesize) {
-            io_->read(chunkId.pData_, WEBP_TAG_SIZE);
-            io_->read(size_buff, WEBP_TAG_SIZE);
-            long size = Exiv2::getULong(size_buff, littleEndian);
+            readOrThrow(*io_, chunkId.pData_, WEBP_TAG_SIZE, Exiv2::kerCorruptedMetadata);
+            readOrThrow(*io_, size_buff, WEBP_TAG_SIZE, Exiv2::kerCorruptedMetadata);
+            const uint32_t size_u32 = Exiv2::getULong(size_buff, littleEndian);
+
+            // Check that `size_u32` is safe to cast to `long`.
+            enforce(size_u32 <= static_cast<size_t>(std::numeric_limits<unsigned int>::max()),
+                    Exiv2::kerCorruptedMetadata);
+            const long size = static_cast<long>(size_u32);
             DataBuf payload(size);
-            io_->read(payload.pData_, payload.size_);
-            byte c;
-            if ( payload.size_ % 2 ) io_->read(&c,1);
+            readOrThrow(*io_, payload.pData_, payload.size_, Exiv2::kerCorruptedMetadata);
+            if ( payload.size_ % 2 ) {
+              byte c;
+              readOrThrow(*io_, &c, 1, Exiv2::kerCorruptedMetadata);
+            }

             /* Chunk with information about features
              used in the file. */
             if (equalsWebPTag(chunkId, WEBP_CHUNK_HEADER_VP8X) && !has_vp8x) {
                 has_vp8x = true;
             }
             if (equalsWebPTag(chunkId, WEBP_CHUNK_HEADER_VP8X) && !has_size) {
+                enforce(size >= 10, Exiv2::kerCorruptedMetadata);
                 has_size = true;
                 byte size_buf[WEBP_TAG_SIZE];

@@ -227,6 +235,7 @@ namespace Exiv2 {
             }
 #endif
             if (equalsWebPTag(chunkId, WEBP_CHUNK_HEADER_VP8) && !has_size) {
+                enforce(size >= 10, Exiv2::kerCorruptedMetadata);
                 has_size = true;
                 byte size_buf[2];

@@ -244,11 +253,13 @@ namespace Exiv2 {

             /* Chunk with with lossless image data. */
             if (equalsWebPTag(chunkId, WEBP_CHUNK_HEADER_VP8L) && !has_alpha) {
+                enforce(size >= 5, Exiv2::kerCorruptedMetadata);
                 if ((payload.pData_[4] & WEBP_VP8X_ALPHA_BIT) == WEBP_VP8X_ALPHA_BIT) {
                     has_alpha = true;
                 }
             }
             if (equalsWebPTag(chunkId, WEBP_CHUNK_HEADER_VP8L) && !has_size) {
+                enforce(size >= 5, Exiv2::kerCorruptedMetadata);
                 has_size = true;
                 byte size_buf_w[2];
                 byte size_buf_h[3];
@@ -276,11 +287,13 @@ namespace Exiv2 {

             /* Chunk with animation frame. */
             if (equalsWebPTag(chunkId, WEBP_CHUNK_HEADER_ANMF) && !has_alpha) {
+                enforce(size >= 6, Exiv2::kerCorruptedMetadata);
                 if ((payload.pData_[5] & 0x2) == 0x2) {
                     has_alpha = true;
                 }
             }
             if (equalsWebPTag(chunkId, WEBP_CHUNK_HEADER_ANMF) && !has_size) {
+                enforce(size >= 12, Exiv2::kerCorruptedMetadata);
                 has_size = true;
                 byte size_buf[WEBP_TAG_SIZE];

@@ -309,16 +322,22 @@ namespace Exiv2 {

         io_->seek(12, BasicIo::beg);
         while ( !io_->eof() && (uint64_t) io_->tell() < filesize) {
-            io_->read(chunkId.pData_, 4);
-            io_->read(size_buff, 4);
+            readOrThrow(*io_, chunkId.pData_, 4, Exiv2::kerCorruptedMetadata);
+            readOrThrow(*io_, size_buff, 4, Exiv2::kerCorruptedMetadata);
+
+            const uint32_t size_u32 = Exiv2::getULong(size_buff, littleEndian);

-            long size = Exiv2::getULong(size_buff, littleEndian);
+            // Check that `size_u32` is safe to cast to `long`.
+            enforce(size_u32 <= static_cast<size_t>(std::numeric_limits<unsigned int>::max()),
+                    Exiv2::kerCorruptedMetadata);
+            const long size = static_cast<long>(size_u32);

             DataBuf payload(size);
-            io_->read(payload.pData_, size);
+            readOrThrow(*io_, payload.pData_, size, Exiv2::kerCorruptedMetadata);
             if ( io_->tell() % 2 ) io_->seek(+1,BasicIo::cur); // skip pad

             if (equalsWebPTag(chunkId, WEBP_CHUNK_HEADER_VP8X)) {
+                enforce(size >= 1, Exiv2::kerCorruptedMetadata);
                 if (has_icc){
                     payload.pData_[0] |= WEBP_VP8X_ICC_BIT;
                 } else {


Fix handling of invalid union data in table-based serializer

Summary:
Fix handling of invalid union data in the table-based serializer. Previously if the input contained duplicate union data, previous active member of the union was overwritten without calling the destructor of the old object, potentially causing a memory leak. In addition to that, if the second piece of data was incomplete the wrong destructor would be called during stack unwinding causing a segfault, data corruption or other undesirable effects.

Fix the issue by clearing the union if there is an active member.

Also fix the type of the data member that holds the active field id (it's `int`, not `FieldID`).

Reviewed By: yfeldblum

Differential Revision: D26440248

fbshipit-source-id: fae9ab96566cf07e14dabe9663b2beb680a01bb4

@@ -356,3 +356,25 @@ TEST(SerializerTest, UnionValueOffsetIsZero) {
   u.set_fieldB({});
   EXPECT_EQ(static_cast<void*>(&u), &*u.fieldB_ref());
 }
+
+TEST(SerializerTest, DuplicateUnionData) {
+  // Test that we can handle invalid serialized input with duplicate and
+  // incomplete union data.
+  const char data[] =
+      "\x0c" // type = TType::T_STRUCT
+      "\x00\x01" // fieldId = 1 (unionField)
+      "\x0b" // type = TType::T_STRING
+      "\x00\x01" // fieldId = 1 (stringField)
+      "\x00\x00\x00\x00" // size = 0
+      "\x00" // end of unionField
+
+      "\x0c" // type = TType::T_STRUCT
+      "\x00\x01" // fieldId = 1 (unionField)
+      "\x13" // type = TType::T_FLOAT
+      "\x00\x02"; // fieldId = 2 (floatField), value is missing
+
+  EXPECT_THROW(
+      BinarySerializer::deserialize<tablebased::TestStructWithUnion>(
+          folly::StringPiece(data, sizeof(data))),
+      std::out_of_range);
+}


Implement handling of escaped slash characters in URL path

Fixes: CVE-2021-29492

Signed-off-by: Yan Avlasov <yavlasov@google.com>

@@ -49,12 +49,14 @@ namespace Http {
   COUNTER(downstream_rq_4xx)                                                                       \
   COUNTER(downstream_rq_5xx)                                                                       \
   COUNTER(downstream_rq_completed)                                                                 \
+  COUNTER(downstream_rq_failed_path_normalization)                                                 \
   COUNTER(downstream_rq_http1_total)                                                               \
   COUNTER(downstream_rq_http2_total)                                                               \
   COUNTER(downstream_rq_http3_total)                                                               \
   COUNTER(downstream_rq_idle_timeout)                                                              \
   COUNTER(downstream_rq_non_relative_path)                                                         \
   COUNTER(downstream_rq_overload_close)                                                            \
+  COUNTER(downstream_rq_redirected_with_normalized_path)                                           \
   COUNTER(downstream_rq_response_before_rq_complete)                                               \
   COUNTER(downstream_rq_rx_reset)                                                                  \
   COUNTER(downstream_rq_timeout)                                                                   \
@@ -466,6 +468,14 @@ class ConnectionManagerConfig {
    * @return LocalReply configuration which supplies mapping for local reply generated by Envoy.
    */
   virtual const LocalReply::LocalReply& localReply() const PURE;
+
+  /**
+   * @return the action HttpConnectionManager should take when receiving client request
+   * with URI path containing %2F, %2f, %5c or %5C sequences.
+   */
+  virtual envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager::
+      PathWithEscapedSlashesAction
+      pathWithEscapedSlashesAction() const PURE;
 };
 } // namespace Http
 } // namespace Envoy


@@ -522,6 +522,103 @@ TEST_F(HttpConnectionManagerImplTest, RouteShouldUseSantizedPath) {
   filter_callbacks_.connection_.raiseEvent(Network::ConnectionEvent::RemoteClose);
 }

+// Paths with escaped slashes rejected with 400 when configured.
+TEST_F(HttpConnectionManagerImplTest, PathWithEscapedSlashesRejected) {
+  path_with_escaped_slashes_action_ = envoy::extensions::filters::network::http_connection_manager::
+      v3::HttpConnectionManager::REJECT_REQUEST;
+  testPathNormalization(
+      TestRequestHeaderMapImpl{{":authority", "host"}, {":path", "/abc%5c../"}, {":method", "GET"}},
+      TestResponseHeaderMapImpl{{":status", "400"}, {"connection", "close"}});
+  EXPECT_EQ(1U, stats_.named_.downstream_rq_failed_path_normalization_.value());
+}
+
+// Paths with escaped slashes redirected when configured.
+TEST_F(HttpConnectionManagerImplTest, PathWithEscapedSlashesRedirected) {
+  path_with_escaped_slashes_action_ = envoy::extensions::filters::network::http_connection_manager::
+      v3::HttpConnectionManager::UNESCAPE_AND_REDIRECT;
+  testPathNormalization(
+      TestRequestHeaderMapImpl{{":authority", "host"}, {":path", "/abc%2f../"}, {":method", "GET"}},
+      TestResponseHeaderMapImpl{{":status", "307"}, {"location", "/abc/../"}});
+  EXPECT_EQ(1U, stats_.named_.downstream_rq_redirected_with_normalized_path_.value());
+}
+
+// Paths with escaped slashes rejected with 400 instead of redirected for gRPC request.
+TEST_F(HttpConnectionManagerImplTest, PathWithEscapedSlashesRejectedIfGRPC) {
+  // This test is slightly weird as it sends gRPC "request" over H/1 client of the
+  // HttpConnectionManagerImplTest. However it is sufficient to test the behavior of path
+  // normalization as it is determined by the content type only.
+  path_with_escaped_slashes_action_ = envoy::extensions::filters::network::http_connection_manager::
+      v3::HttpConnectionManager::UNESCAPE_AND_REDIRECT;
+  testPathNormalization(TestRequestHeaderMapImpl{{":authority", "host"},
+                                                 {":path", "/abc%2fdef"},
+                                                 {":method", "GET"},
+                                                 {"content-type", "application/grpc"}},
+                        TestResponseHeaderMapImpl{{":status", "200"},
+                                                  {"connection", "close"},
+                                                  {"grpc-status", "13"},
+                                                  {"content-type", "application/grpc"}});
+  EXPECT_EQ(1U, stats_.named_.downstream_rq_failed_path_normalization_.value());
+}
+
+// Test that requests with escaped slashes are redirected when configured. Redirection
+// occurs after Chromium URL normalization or merge slashes operations.
+TEST_F(HttpConnectionManagerImplTest, EscapedSlashesRedirectedAfterOtherNormalizations) {
+  normalize_path_ = true;
+  merge_slashes_ = true;
+  path_with_escaped_slashes_action_ = envoy::extensions::filters::network::http_connection_manager::
+      v3::HttpConnectionManager::UNESCAPE_AND_REDIRECT;
+  // Both Chromium URL normalization and merge slashes should happen if request is redirected
+  // due to escaped slash sequences.
+  testPathNormalization(TestRequestHeaderMapImpl{{":authority", "host"},
+                                                 {":path", "/abc%2f../%5cdef//"},
+                                                 {":method", "GET"}},
+                        TestResponseHeaderMapImpl{{":status", "307"}, {"location", "/def/"}});
+  EXPECT_EQ(1U, stats_.named_.downstream_rq_redirected_with_normalized_path_.value());
+}
+
+TEST_F(HttpConnectionManagerImplTest, AllNormalizationsWithEscapedSlashesForwarded) {
+  setup(false, "");
+  // Enable path sanitizer
+  normalize_path_ = true;
+  merge_slashes_ = true;
+  path_with_escaped_slashes_action_ = envoy::extensions::filters::network::http_connection_manager::
+      v3::HttpConnectionManager::UNESCAPE_AND_FORWARD;
+  const std::string original_path = "/x/%2E%2e/z%2f%2Fabc%5C../def";
+  const std::string normalized_path = "/z/def";
+
+  auto* filter = new MockStreamFilter();
+
+  EXPECT_CALL(filter_factory_, createFilterChain(_))
+      .WillOnce(Invoke([&](FilterChainFactoryCallbacks& callbacks) -> void {
+        callbacks.addStreamDecoderFilter(StreamDecoderFilterSharedPtr{filter});
+      }));
+
+  EXPECT_CALL(*filter, decodeComplete());
+  EXPECT_CALL(*filter, decodeHeaders(_, true))
+      .WillRepeatedly(Invoke([&](RequestHeaderMap& header_map, bool) -> FilterHeadersStatus {
+        EXPECT_EQ(normalized_path, header_map.getPathValue());
+        return FilterHeadersStatus::StopIteration;
+      }));
+
+  EXPECT_CALL(*filter, setDecoderFilterCallbacks(_));
+
+  EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance&) -> Http::Status {
+    decoder_ = &conn_manager_->newStream(response_encoder_);
+    RequestHeaderMapPtr headers{new TestRequestHeaderMapImpl{
+        {":authority", "host"}, {":path", original_path}, {":method", "GET"}}};
+    decoder_->decodeHeaders(std::move(headers), true);
+    return Http::okStatus();
+  }));
+
+  // Kick off the incoming data.
+  Buffer::OwnedImpl fake_input("1234");
+  conn_manager_->onData(fake_input, false);
+
+  EXPECT_CALL(*filter, onStreamComplete());
+  EXPECT_CALL(*filter, onDestroy());
+  filter_callbacks_.connection_.raiseEvent(Network::ConnectionEvent::RemoteClose);
+}
+
 TEST_F(HttpConnectionManagerImplTest, RouteOverride) {
   setup(false, "");