Migrate ocaml ppx context

astlib/migrate_501_502.ml

@@ -7,13 +7,59 @@ module To = Ast_502
     attribute is not found. *)
 let extract_attr name (attrs : Ast_501.Parsetree.attributes) =
   let rec loop acc = function
-    | [] -> (false, List.rev acc)
-    | { Ast_501.Parsetree.attr_name = { txt; _ }; _ } :: q when txt = name ->
-        (true, List.rev_append acc q)
+    | [] -> (None, List.rev acc)
+    | { Ast_501.Parsetree.attr_name = { txt; _ }; attr_payload; _ } :: q
+      when txt = name ->
+        (Some attr_payload, List.rev_append acc q)
     | hd :: tl -> loop (hd :: acc) tl
   in
   loop [] attrs
 
+let migrate_ppx_context_load_path expr =
+  let open Ast_501.Parsetree in
+  let payload, other_attrs =
+    extract_attr "ppxlib.migration.load_path" expr.pexp_attributes
+  in
+  match payload with
+  | None ->
+      let pexp_desc =
+        Pexp_construct ({ txt = Lident "[]"; loc = expr.pexp_loc }, None)
+      in
+      let hidden = { expr with pexp_desc; pexp_attributes = [] } in
+      let visible = expr in
+      {
+        expr with
+        pexp_attributes = other_attrs;
+        pexp_desc = Pexp_tuple [ visible; hidden ];
+      }
+  | Some (PStr [ { pstr_desc = Pstr_eval (expr, []); _ } ]) -> expr
+  | Some _ -> invalid_arg "Invalid ppxlib.migration.load_path payload"
+
+let migrate_ppx_context_fields fields =
+  List.map
+    (fun (lident_loc, expr) ->
+      match lident_loc.Ast_501.Asttypes.txt with
+      | Longident.Lident "load_path" ->
+          (lident_loc, migrate_ppx_context_load_path expr)
+      | _ -> (lident_loc, expr))
+    fields
+
+let migrate_ppx_context_payload payload =
+  let open Ast_501.Parsetree in
+  match payload with
+  | PStr
+      [
+        ({
+           pstr_desc =
+             Pstr_eval
+               (({ pexp_desc = Pexp_record (fields, None) } as expr), attributes);
+         } as stri);
+      ] ->
+      let new_fields = migrate_ppx_context_fields fields in
+      let new_expr = { expr with pexp_desc = Pexp_record (new_fields, None) } in
+      PStr [ { stri with pstr_desc = Pstr_eval (new_expr, attributes) } ]
+  | _ -> payload
+
 let rec copy_toplevel_phrase :
     Ast_501.Parsetree.toplevel_phrase -> Ast_502.Parsetree.toplevel_phrase =
   function
@@ -514,7 +560,11 @@ and copy_attribute : Ast_501.Parsetree.attribute -> Ast_502.Parsetree.attribute
      } ->
   {
     Ast_502.Parsetree.attr_name = copy_loc (fun x -> x) attr_name;
-    Ast_502.Parsetree.attr_payload = copy_payload attr_payload;
+    Ast_502.Parsetree.attr_payload =
+      (match attr_name.txt with
+      | "ocaml.ppx.context" ->
+          copy_payload (migrate_ppx_context_payload attr_payload)
+      | _ -> copy_payload attr_payload);
     Ast_502.Parsetree.attr_loc = copy_location attr_loc;
   }
 

astlib/migrate_502_501.ml

@@ -13,6 +13,71 @@ let mk_ghost_attr name =
     attr_loc = Location.none;
   }
 
+let rec concat_list_lit left right =
+  let open Ast_502.Parsetree in
+  let open Ast_502.Asttypes in
+  match (left.pexp_desc, right.pexp_desc) with
+  | _, Pexp_construct ({ txt = Lident "[]"; _ }, _) -> left
+  | Pexp_construct ({ txt = Lident "[]"; _ }, _), _ -> right
+  | ( Pexp_construct
+        ( { txt = Lident "::"; loc },
+          Some ({ pexp_desc = Pexp_tuple [ hd; tl ]; _ } as arg_expr) ),
+      _ ) ->
+      {
+        left with
+        pexp_desc =
+          Pexp_construct
+            ( { txt = Lident "::"; loc },
+              Some
+                {
+                  arg_expr with
+                  pexp_desc = Pexp_tuple [ hd; concat_list_lit tl right ];
+                } );
+      }
+  | _ -> invalid_arg "Invalid ocaml.ppx.context's load_path"
+
+let migrate_ppx_context_load_path expr =
+  let open Ast_502.Parsetree in
+  let loc = expr.pexp_loc in
+  match expr.pexp_desc with
+  | Pexp_tuple [ visible; hidden ] ->
+      let migration_attr =
+        {
+          attr_name = { Location.txt = "ppxlib.migration.load_path"; loc };
+          attr_loc = loc;
+          attr_payload =
+            PStr [ { pstr_loc = loc; pstr_desc = Pstr_eval (expr, []) } ];
+        }
+      in
+      let expr' = concat_list_lit visible hidden in
+      { expr' with pexp_attributes = migration_attr :: expr.pexp_attributes }
+  | _ -> expr
+
+let migrate_ppx_context_fields fields =
+  List.map
+    (fun (lident_loc, expr) ->
+      match lident_loc.Ast_502.Asttypes.txt with
+      | Longident.Lident "load_path" ->
+          (lident_loc, migrate_ppx_context_load_path expr)
+      | _ -> (lident_loc, expr))
+    fields
+
+let migrate_ppx_context_payload payload =
+  let open Ast_502.Parsetree in
+  match payload with
+  | PStr
+      [
+        ({
+           pstr_desc =
+             Pstr_eval
+               (({ pexp_desc = Pexp_record (fields, None) } as expr), attributes);
+         } as stri);
+      ] ->
+      let new_fields = migrate_ppx_context_fields fields in
+      let new_expr = { expr with pexp_desc = Pexp_record (new_fields, None) } in
+      PStr [ { stri with pstr_desc = Pstr_eval (new_expr, attributes) } ]
+  | _ -> payload
+
 let rec copy_toplevel_phrase :
     Ast_502.Parsetree.toplevel_phrase -> Ast_501.Parsetree.toplevel_phrase =
   function
@@ -511,7 +576,11 @@ and copy_attribute : Ast_502.Parsetree.attribute -> Ast_501.Parsetree.attribute
      } ->
   {
     Ast_501.Parsetree.attr_name = copy_loc (fun x -> x) attr_name;
-    Ast_501.Parsetree.attr_payload = copy_payload attr_payload;
+    Ast_501.Parsetree.attr_payload =
+      (match attr_name.txt with
+      | "ocaml.ppx.context" ->
+          copy_payload (migrate_ppx_context_payload attr_payload)
+      | _ -> copy_payload attr_payload);
     Ast_501.Parsetree.attr_loc = copy_location attr_loc;
   }
 

test/driver/ocaml-ppx-context-load-path-migration/driver.ml

@@ -0,0 +1,92 @@
+module To_before_502 =
+  Ppxlib_ast.Convert (Ppxlib_ast.Js) (Ppxlib_ast__.Versions.OCaml_501)
+
+module From_before_502 =
+  Ppxlib_ast.Convert (Ppxlib_ast__.Versions.OCaml_501) (Ppxlib_ast.Js)
+
+module Before_502_to_ocaml =
+  Ppxlib_ast.Convert
+    (Ppxlib_ast__.Versions.OCaml_501)
+    (Ppxlib_ast.Compiler_version)
+
+module OCaml_501 = Ppxlib_ast__.Versions.OCaml_501.Ast
+
+let rec unfold_list_lit x next =
+  let open OCaml_501.Parsetree in
+  let open Astlib.Longident in
+  match next.pexp_desc with
+  | Pexp_construct ({ txt = Lident "[]"; _ }, None) -> [ x ]
+  | Pexp_construct
+      ( { txt = Lident "::"; _ },
+        Some { pexp_desc = Pexp_tuple [ elm; rest ]; _ } ) ->
+      x :: unfold_list_lit elm rest
+  | _ -> invalid_arg "list_lit"
+
+(* Only deals with the basic blocks needed for ocaml.ppx.context *)
+let rec basic_expr_to_string expr =
+  let open OCaml_501.Parsetree in
+  let open Astlib.Longident in
+  match expr.pexp_desc with
+  | Pexp_constant (Pconst_string (s, _, None)) -> Printf.sprintf "%S" s
+  | Pexp_ident { txt = Lident name; _ } -> name
+  | Pexp_tuple l ->
+      let strs = List.map basic_expr_to_string l in
+      "(" ^ String.concat ", " strs ^ ")"
+  | Pexp_construct ({ txt = Lident s; _ }, None) -> s
+  | Pexp_construct
+      ( { txt = Lident "::"; _ },
+        Some { pexp_desc = Pexp_tuple [ elm; rest ]; _ } ) ->
+      let exprs = unfold_list_lit elm rest in
+      let strs = List.map basic_expr_to_string exprs in
+      "[" ^ String.concat "; " strs ^ "]"
+  | _ -> invalid_arg "basic_expr_to_string"
+
+let print_field (lident_loc, expr) =
+  match lident_loc with
+  | { OCaml_501.Asttypes.txt = Astlib.Longident.Lident name; _ } ->
+      Printf.printf "    %s: %s;\n" name (basic_expr_to_string expr)
+  | _ -> ()
+
+let print_ocaml_ppx_context stri =
+  let open OCaml_501.Parsetree in
+  match stri.pstr_desc with
+  | Pstr_attribute
+      {
+        attr_payload =
+          PStr
+            [
+              {
+                pstr_desc =
+                  Pstr_eval ({ pexp_desc = Pexp_record (fields, None); _ }, _);
+                _;
+              };
+            ];
+        _;
+      } ->
+      Printf.printf "[@@@ocaml.ppx.context\n";
+      Printf.printf "  {\n";
+      List.iter print_field fields;
+      Printf.printf "  }\n";
+      Printf.printf "]\n"
+  | _ -> ()
+
+let is_ppx_context stri =
+  let open OCaml_501.Parsetree in
+  match stri.pstr_desc with
+  | Pstr_attribute
+      { attr_name = { OCaml_501.Asttypes.txt = "ocaml.ppx.context"; _ }; _ } ->
+      true
+  | _ -> false
+
+let impl _ctxt str =
+  let before_502_ast = To_before_502.copy_structure str in
+  let ppx_context = List.find is_ppx_context before_502_ast in
+  Printf.printf "ocaml.ppx.context before 5.02:\n";
+  print_ocaml_ppx_context ppx_context;
+  let round_trip = Before_502_to_ocaml.copy_structure_item ppx_context in
+  Printf.printf "ocaml.ppx.context round tripped:\n";
+  Ocaml_common.Pprintast.structure_item Format.std_formatter round_trip;
+  str
+
+let () = Ppxlib.Driver.V2.register_transformation ~impl "ocaml.ppx.context-test"
+let () = Ppxlib.Driver.standalone ()

test/driver/ocaml-ppx-context-load-path-migration/dune

@@ -0,0 +1,11 @@
+(executable
+ (name driver)
+ (enabled_if
+  (>= %{ocaml_version} "5.2"))
+ (libraries ppxlib ppxlib.ast ppxlib.astlib ocaml-compiler-libs.common
+   compiler-libs.common))
+
+(cram
+ (enabled_if
+  (>= %{ocaml_version} "5.2"))
+ (deps driver.exe))

test/driver/ocaml-ppx-context-load-path-migration/run.t

@@ -0,0 +1,73 @@
+In 5.2 the format of ocaml.ppx.context load_path changed.
+To ensure compat, we defined migration for ocaml.ppx.context attributes
+
+We write such an attribute to an .ml file. The compiler will add its own
+and it should be consumed by the driver but our handwritten attribute will
+be migrated as well and should remain in the AST.
+  $ cat > test.ml << EOF
+  > let x = 1
+  > [@@@ocaml.ppx.context
+  >   {
+  >     tool_name = "ocaml";
+  >     include_dirs = ["foo"];
+  >     hidden_include_dirs = [];
+  >     load_path = (["foo"; "bar"], ["baz"]);
+  >     open_modules = [];
+  >     for_package = None;
+  >     debug = true;
+  >     use_threads = false;
+  >     use_vmthreads = false;
+  >     recursive_types = false;
+  >     principal = false;
+  >     transparent_modules = false;
+  >     unboxed_types = false;
+  >     unsafe_string = false;
+  >     cookies = []
+  >   }]
+  > EOF
+
+We then run a custom driver that will read our ast, migrate it back to 5.01,
+pretty print the ocaml.ppx.context, convert it back to the latest version and
+pretty print it again. This last, round-tripped version should be identical to
+the one above.
+
+  $ ./driver.exe --impl test.ml -o ignore.ml
+  ocaml.ppx.context before 5.02:
+  [@@@ocaml.ppx.context
+    {
+      tool_name: "ocaml";
+      include_dirs: ["foo"];
+      hidden_include_dirs: [];
+      load_path: ["foo"; "bar"; "baz"];
+      open_modules: [];
+      for_package: None;
+      debug: true;
+      use_threads: false;
+      use_vmthreads: false;
+      recursive_types: false;
+      principal: false;
+      transparent_modules: false;
+      unboxed_types: false;
+      unsafe_string: false;
+      cookies: [];
+    }
+  ]
+  ocaml.ppx.context round tripped:
+  [@@@ocaml.ppx.context
+    {
+      tool_name = "ocaml";
+      include_dirs = ["foo"];
+      hidden_include_dirs = [];
+      load_path = (["foo"; "bar"], ["baz"]);
+      open_modules = [];
+      for_package = None;
+      debug = true;
+      use_threads = false;
+      use_vmthreads = false;
+      recursive_types = false;
+      principal = false;
+      transparent_modules = false;
+      unboxed_types = false;
+      unsafe_string = false;
+      cookies = []
+    }]

test/driver/standalone-supports-old-binary-ast/dune

@@ -0,0 +1,19 @@
+(executable
+ (name identity_standalone)
+ (libraries ppxlib)
+ (modules identity_standalone))
+
+(executable
+ (name print_magic_number)
+ (libraries astlib)
+ (modules print_magic_number))
+
+(cram
+ (enabled_if
+  (or
+   (= %{system} linux)
+   (= %{system} linux_elf)
+   (= %{system} elf)
+   (= %{system} linux_eabihf)
+   (= %{system} linux_eabi)))
+ (deps identity_standalone.exe print_magic_number.exe))

test/driver/standalone-supports-old-binary-ast/identity_standalone.ml

@@ -0,0 +1 @@
+let () = Ppxlib.Driver.standalone ()

test/driver/standalone-supports-old-binary-ast/print_magic_number.ml

@@ -0,0 +1,5 @@
+let magic_length = String.length Astlib.Config.ast_impl_magic_number
+let buf = Bytes.create magic_length
+let len = input stdin buf 0 magic_length
+let s = Bytes.sub_string buf 0 len
+let () = Printf.printf "Magic number: %s" s

test/driver/standalone-supports-old-binary-ast/test.t/run.t

@@ -0,0 +1,10 @@
+Binary AST's of any by ppxlib supported OCaml version are supported.
+The version is preserved.
+
+  $ cat 406_binary_ast | ../print_magic_number.exe
+  Magic number: Caml1999N022
+
+  $ ../identity_standalone.exe --intf 406_binary_ast -o transformed --dump-ast
+
+  $ ../print_magic_number.exe < transformed
+  Magic number: Caml1999N022