[Lunar-commits] CVS: theedge/var/lib/lunar/functions check.lunar, 1.13, 1.14 depends.lunar, 1.29, 1.30 modules.lunar, 1.31, 1.32 moonbase.lunar, 1.15, 1.16

Auke Kok sofar at lunar-linux.org
Sun Mar 7 23:32:33 GMT 2004


Update of /var/cvs/lunar/theedge/var/lib/lunar/functions
In directory dbguin.lunar-linux.org:/tmp/cvs-serv11821/var/lib/lunar/functions

Modified Files:
	check.lunar depends.lunar modules.lunar moonbase.lunar 
Log Message:
Fixes for:
- dependency sorting algorithm, completely rewritten from scratch, implemented using a depends.cache file which speeds up all dependency work immensely. The time needed to create the cache might still be a little long tho (2:30 uncached on my box). sorting time for entire moonbase is about 2:30 now, mostly spending in the fgrep. sorting time on 400 modules is < 30 seconds.
- double questions removed
- moonbase index creation is now more atomic than it was, it's generated in a tmpfile, then pivoted to the real location, same for depends.cache (crtl-c in the creation will not damage it)
- lrm fix that caused files in /boot to be deleted. as far as I can see this was still an TODO left open by kongar&&those-before-him (ouch). tested and works.
- added $DEPENDS_CACHE var in etc/lunar/config


Index: check.lunar
===================================================================
RCS file: /var/cvs/lunar/theedge/var/lib/lunar/functions/check.lunar,v
retrieving revision 1.13
retrieving revision 1.14
diff -u -d -r1.13 -r1.14
--- check.lunar	29 Dec 2003 18:09:42 -0000	1.13
+++ check.lunar	7 Mar 2004 23:32:31 -0000	1.14
@@ -295,7 +295,7 @@
   if [ -n "$1" ] ; then
     LIST="$@"
   else
-    LIST=$(cat $MODULE_STATUS | cut -d: -f1 | sort )
+    LIST=$(grep -v moonbase: $MODULE_STATUS | cut -d: -f1 | sort )
   fi
 
   for MODULE in $LIST ; do

Index: depends.lunar
===================================================================
RCS file: /var/cvs/lunar/theedge/var/lib/lunar/functions/depends.lunar,v
retrieving revision 1.29
retrieving revision 1.30
diff -u -d -r1.29 -r1.30
--- depends.lunar	5 Mar 2004 20:46:23 -0000	1.29
+++ depends.lunar	7 Mar 2004 23:32:31 -0000	1.30
@@ -16,9 +16,7 @@
 #                                                          #
 # Copyrighted Kagan Kongar 2002 under GPLv2                #
 #                                                          #
-############################################################
-#                                                          #
-# Merged in more depends routines - sofar - 20030305       # 
+# Copyright Auke Kok 2004 under GPLv2                      #
 #                                                          #
 ############################################################
 
@@ -29,38 +27,28 @@
 function find_depends() {
   debug_msg "function find_depends ($@)"
 
-  # as always, use private depends() and optional_depends() functions
-  depends () {
-  debug_msg "  depends  ($@)"
-    if ! $(echo $FOUND_DEPENDS | grep -qw $1) ; then
-      export FOUND_DEPENDS="$FOUND_DEPENDS $1"
-      find_depends $1
-      echo $1
-    fi
-  }
-
-  optional_depends() {
-  debug_msg "  optional_depends ($@)"
-    if ! $(echo $FOUND_DEPENDS | grep -qw $1) ; then
-      if module_installed $1 ; then
-        export FOUND_DEPENDS="$FOUND_DEPENDS $1"
-        find_depends $1
-        echo $1
+  find_depends_intern() {
+    grep "^$1:" $DEPENDS_CACHE | while read LINE ; do
+      DEP=$(echo $LINE | cut -d: -f2)
+      # this is our shortcut out:
+      if ! grep -qx "$DEP" $TMP_FDEPS ; then
+      debug_msg "$DEP"
+        echo "$DEP" >> $TMP_FDEPS
+        STATE=$(echo $LINE | cut -d: -f3)
+        if [ "$STATE" == "required" ] ; then
+          echo $DEP
+          find_depends_intern $DEP
+        elif module_installed $DEP ; then
+          echo $DEP
+          find_depends_intern $DEP
+        fi
       fi
-    fi
+    done
   }
-  
-  export FOUND_DEPENDS="$FOUND_DEPENDS $1"
-  if ! run_details $1 &> /dev/null ; then
-    exit 1
-  fi
-
-  if [ -e "$MODULE_CONFIG" ] ; then
-    . "$MODULE_CONFIG"
-  fi
-
-  run_module_file $MODULE DEPENDS
 
+  TMP_FDEPS=$(temp_create "found.depends")
+  find_depends_intern $1
+  temp_destroy $TMP_FDEPS
 }
 
 
@@ -70,36 +58,36 @@
 sort_by_dependency() {
   debug_msg "sort_by_dependency ($@)"
 
-  unset INCLUDED
+  TMP_LIST=$(temp_create "deptree.in")
+  TMP_ALL=$(temp_create "deptree.all")
 
-  recurse() {
-    for MOD in $* ; do
-      if echo "$INCLUDED" | grep -qFx "$MOD" ; then
-        debug_msg "Cutoff: $MOD"
-      else
-        LIST=$(grep ^$MOD: $DEPENDS_STATUS | grep ":on:" | cut -d: -f2)
-        for DEP in $LIST ; do
-          echo $MOD $DEP
-        done
-        INCLUDED=$INCLUDED$'\n'$MOD
-        recurse $LIST
-      fi
-    done
-  }
+  for M in $* ; do
+    echo "$M" >> $TMP_LIST
+  done
 
-  ALL=$(recurse $* | tsort | tac)
-  for DEP in $ALL ; do
-    if echo " $* " | grep -q " $DEP " ; then
-      echo $DEP
+  # tsort the existing dep relations in all of moonbase
+  cat $DEPENDS_CACHE | awk -F: '{print $1" "$2}' | tsort 2> /dev/null | tac > $TMP_ALL
+  # append all modules that do not have a DEPENDS file at all at the end
+  # note that this is logically WRONG, but it actually will help with
+  # unincluded depends, and therefore is *better* behaviour
+  for M in $* ; do
+    if ! grep -q -x $M $TMP_ALL ; then
+      echo $M >> $TMP_ALL
     fi
   done
+
+  # now reverse grep over the files:
+  cat $TMP_ALL | grep -x -f $TMP_LIST
+  
+  temp_destroy $TMP_LIST
+  temp_destroy $TMP_ALL
 }
 
 
 is_depends()  {
   debug_msg "is_depends ($@)"
   # Is $1 a previously selected dependency of any module.
-  return $(cat $DEPENDS_STATUS | cut -d: -f2- | grep -q "^$1:on:")
+  return $(grep -q ":$1:on:" $DEPENDS_STATUS)
 }
                                                                                 
                                                                                 
@@ -327,4 +315,35 @@
 }
 
 
+create_depends_cache() {
+  debug_msg "create_depends_cache($@)"
+
+  if [ $MODULE_INDEX -nt $DEPENDS_CACHE ] ; then
+    verbose_msg "Generating a new depends cache..."
+    TMP_DEP_CACHE=$(temp_create "depends.cache")
+
+    # you guessed it: local decls.
+    depends() {
+      echo "$MODULE:$1:required:::"
+    }
+
+    optional_depends() {
+      echo "$MODULE:$1:optional:$2:$3:$4"
+    }
+
+    # yeah, this sucks:
+    message() {
+      :
+    }
 
+    # fast method for re-creating the depends.cache, might take long though
+    for DEPFILE in $(find $MOONBASE -type f -name DEPENDS) ; do
+      MODULE=$(echo $DEPFILE | sed -e "s:$MOONBASE/::g" -e "s:/DEPENDS::g" | cut -d/ -f2-) . $DEPFILE
+    done > $TMP_DEP_CACHE &&
+    install -m644 $TMP_DEP_CACHE $DEPENDS_CACHE 
+  
+    temp_destroy $TMP_DEP_CACHE
+  else
+    verbose_msg "Skipping depends cache regeneration"
+  fi
+}

Index: modules.lunar
===================================================================
RCS file: /var/cvs/lunar/theedge/var/lib/lunar/functions/modules.lunar,v
retrieving revision 1.31
retrieving revision 1.32
diff -u -d -r1.31 -r1.32
--- modules.lunar	23 Feb 2004 21:15:53 -0000	1.31
+++ modules.lunar	7 Mar 2004 23:32:31 -0000	1.32
@@ -30,21 +30,8 @@
 list_sections() {
   debug_msg "list_sections ($@)"
   check_module_index
-  SECTIONS=$(grep -v ^MOONBASE_MD5 $MODULE_INDEX  2>/dev/null | \
-             cut -d : -f 2-2 | sort | uniq)
-  if [ -n "$SECTIONS" ] ; then
-    echo "$SECTIONS"
-    return 0
-  fi
-
-  SECTIONS=$(ls $MOONBASE | grep -v -e "ChangeLog" -e "CVS" -e "COPYING" \
-            -e "LICENSE" -e "README")
-  if [ -n "$SECTIONS" ] ; then
-    echo "$SECTIONS"
-    return 0
-  fi
-
-  return 1
+  grep -v ^MOONBASE_MD5: $MODULE_INDEX | cut -d: -f2 | sort | uniq
+  return 0
 }
 
 
@@ -58,12 +45,12 @@
     exit 1
   fi
 
-  if [ ! -d $MOONBASE/$1 ] ; then
+  if ! grep -q ":$1" $MODULE_INDEX ; then
     error_message "${PROBLEM_COLOR}list_modules(): no such section \"$1\"!${DEFAULT_COLOR}"
     exit 1
   fi
 
-  ls -d $MOONBASE/$1/* $MOONBASE/$1/*/[0-9]* 2> /dev/null | sed "s:$MOONBASE/$1/::" | grep -v -e "ChangeLog" -e "CVS" -e "COPYING" -e "LICENSE" -e "README"
+  grep ":$1" $MODULE_INDEX | cut -d: -f1
   return 0
 }
 
@@ -93,8 +80,12 @@
 # purpose : created an index file of module:section pair list
 create_module_index() {
   debug_msg "create_module_index ($@)"
+  verbose_msg "Updating module index file..."
   # make sure it exists before trying to see it's writeable
-  touch $MODULE_INDEX &> /dev/null
+  # this also assures that depends.cache gets remade
+  if [ ! -f $MODULE_INDEX ] ; then
+    touch $MODULE_INDEX &> /dev/null
+  fi
   
   # silently fail if we cannot write to $MODULE_INDEX, it is okay
   # for this to fail in case we are a mere user using lvu's search
@@ -102,17 +93,23 @@
   if [ ! -w "$MODULE_INDEX" ] ; then
     return 0
   fi
+ 
+  # doing this allows us to ctrl-C the process without breaking the
+  # index file
+  TMP_INDEX=$(temp_create "module.index")
 
-  lock_file $MODULE_INDEX || return 1
-
-  rm -f $MODULE_INDEX 2>/dev/null
+  # this *really* is the fastest way to do it, no guarantees, we
+  # do have to make sure MOONBASE is coherent and tidy though
+  echo MOONBASE_MD5:$(set_moonbase_md5) > $TMP_INDEX
+  find $MOONBASE -type f -name DETAILS | sed "s:$MOONBASE/::g" | sed 's|/|:|' | sed 's:/DETAILS::g' | awk -F: '{print $2":"$1}' >> $TMP_INDEX
 
-  echo MOONBASE_MD5:$(set_moonbase_md5) > $MODULE_INDEX
-  for SECTION in $(list_sections) ; do
-    list_modules $SECTION | sed "s/$/:$SECTION/" >> $MODULE_INDEX
-  done
+  # this should be safe enough:
+  lock_file $MODULE_INDEX &&
+  install -m644 $TMP_INDEX $MODULE_INDEX
 
+  # do not forget to do these at any time:
   unlock_file $MODULE_INDEX
+  temp_destroy $TMP_INDEX
 }  
 
 
@@ -121,8 +118,9 @@
 # purpose : checks if the index is up-to-date regarding to moonbase
 function check_module_index() {
   debug_msg "check_module_index ($@)"
-  if [ $(get_moonbase_md5) != $(set_moonbase_md5) ] ; then
+  if [ -n "$(find $MOONBASE -type f -cnewer $MODULE_INDEX)" ] ; then
     create_module_index
+    create_depends_cache
     return 0
   else
     return 1

Index: moonbase.lunar
===================================================================
RCS file: /var/cvs/lunar/theedge/var/lib/lunar/functions/moonbase.lunar,v
retrieving revision 1.15
retrieving revision 1.16
diff -u -d -r1.15 -r1.16
--- moonbase.lunar	23 Feb 2004 18:26:56 -0000	1.15
+++ moonbase.lunar	7 Mar 2004 23:32:31 -0000	1.16
@@ -41,10 +41,6 @@
 
   push_uniq_id
 
-  # Cache our module index before moonbase is removed.
-  TMP_MODULE_INDEX=$(temp_create "module-index")
-  cp $MODULE_INDEX $TMP_MODULE_INDEX
-
   if download_module $MODULE ; then
     echo -e "${MESSAGE_COLOR}Preparing to install ${FILE_COLOR}${SOURCE}" \
             "${DEFAULT_COLOR}${MESSAGE_COLOR}...${DEFAULT_COLOR}"       &&
@@ -72,14 +68,8 @@
       done >> $MD5SUM_LOGS/$MODULE-$VERSION
     
       add_module $MODULE installed $VERSION $(du -hs $SYSTEM_MOONBASE | cut -f1)
-   
       # get ready to regenerate the module index cache file
-      if [ -f "$MODULE_INDEX" ]; then
-        rm -f $MODULE_INDEX
-      fi
-      create_module_index
-      echo -e "${MESSAGE_COLOR}Created ${FILE_COLOR}${MODULE_INDEX}" \
-              "${DEFAULT_COLOR}${MESSAGE_COLOR}${DEFAULT_COLOR}"
+      check_module_index
       display_moonbase_changes
     fi
   else



More information about the Lunar-commits mailing list